From mboxrd@z Thu Jan 1 00:00:00 1970 From: "XU Liang" Subject: Re: =?utf-8?q?=5BPATCH_v7=5D_eal=3A_map_PCI_memory_resourc?= =?utf-8?q?es_after_hugepages?= Date: Tue, 11 Nov 2014 11:53:22 +0800 Message-ID: References: <1415347284-15468-1-git-send-email-liang.xu@cinfotech.cn> <1415619272-8281-1-git-send-email-anatoly.burakov@intel.com>, C6ECDF3AB251BE4894318F4E4512369780C07EEB@IRSMSX109.ger.corp.intel.com Reply-To: XU Liang Mime-Version: 1.0 Content-Type: text/plain; charset="UTF-8" Content-Transfer-Encoding: quoted-printable To: "=?UTF-8?B?QnVyYWtvdiwgQW5hdG9seQ==?=" , "dev-VfR2kkLFssw@public.gmane.org" Return-path: In-Reply-To: C6ECDF3AB251BE4894318F4E4512369780C07EEB-kPTMFJFq+rHjxeytcECX8bfspsVTdybXVpNB7YpNyf8@public.gmane.org List-Id: patches and discussions about DPDK List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces-VfR2kkLFssw@public.gmane.org Sender: "dev" I had finished some tests. The patch works fine. My tests are included :* sing= le process=C2=A0 + uio + vfio * single process=C2=A0 + uio + vfio + base-virta= ddr * multiple processes + uio + vfio *=C2=A0multiple processes + uio + vfio += base-virtaddr My unlucky multiple process application still got error without= =C2=A0base-virtaddr when initial hugepages. See the attchments: primary.txt an= d secondary.txt.With=C2=A0base-virtaddr the patch worked, both hugepages and p= ci resources were mapped into base-virtaddr, My application is happy.=C2=A0See= the attchments: base-virtaddr_primary.txt and base-virtaddr_secondary.txt. -= -----------------------------------------------------------------From:Burakov,= Anatoly Time:2014 Nov 10 (Mon) 21 : 34To:Burakov, = Anatoly , dev-VfR2kkLFssw@public.gmane.org Subject:Re: [d= pdk-dev] [PATCH v7] eal: map PCI memory resources after hugepages=0ANak, there= are issues with the patch. There is another patch already, but I'll submit it= whenever Liang verifies it works with his setup.=0A=0AThanks,=0AAnatoly=0A=0A= -----Original Message-----=0AFrom: dev [mailto:dev-bounces-VfR2kkLFssw@public.gmane.org] On Behalf= Of Anatoly Burakov=0ASent: Monday, November 10, 2014 11:35 AM=0ATo: dev@dpdk.= org=0ASubject: [dpdk-dev] [PATCH v7] eal: map PCI memory resources after hugep= ages=0A=0AMulti-process DPDK application must mmap hugepages and pci resources= =0Ainto the same virtual address space. By default the virtual addresses=0Aare= chosen by the primary process automatically when calling the mmap.=0ABut some= times the chosen virtual addresses aren't usable in secondary=0Aprocess - for = example, secondary process is linked with more libraries=0Athan primary proces= s, and the library occupies the same address space=0Athat the primary process = has requested for PCI mappings.=0A=0AThis patch makes EAL map PCI BARs right a= fter the hugepages (instead of=0Alocation chosen by mmap) in virtual memory.=0A= =0ASigned-off-by: Anatoly Burakov =0ASigned-off-by:= Liang Xu =0A---=0A lib/librte_eal/linuxapp/eal/eal_pci= .c | 19 +++++++++++++++++++=0A lib/librte_eal/linuxapp/eal/eal_pc= i_uio.c | 9 ++++++++-=0A lib/librte_eal/linuxapp/eal/eal_pci_vfio.c = | 13 +++++++++++--=0A lib/librte_eal/linuxapp/eal/include/eal_pci_init= .h | 6 ++++++=0A 4 files changed, 44 insertions(+), 3 deletions(-)=0A=0Adiff = --git a/lib/librte_eal/linuxapp/eal/eal_pci.c b/lib/librte_eal/linuxapp/eal/ea= l_pci.c=0Aindex 5fe3961..dae8739 100644=0A--- a/lib/librte_eal/linuxapp/eal/ea= l_pci.c=0A+++ b/lib/librte_eal/linuxapp/eal/eal_pci.c=0A@@ -97,6 +97,25 @@ err= or:=0A return -1;=0A }=0A =0A+void *=0A+pci_find_max_end_va(void)=0A+{=0A+ co= nst struct rte_memseg *seg =3D rte_eal_get_physmem_layout();=0A+ const struct = rte_memseg *last =3D seg;=0A+ unsigned i =3D 0;=0A+=0A+ for (i =3D 0; i < RTE_= MAX_MEMSEG; i++, seg++) {=0A+ if (seg->addr =3D=3D NULL)=0A+ break;=0A+=0A+= if (seg->addr > last->addr)=0A+ last =3D seg;=0A+=0A+ }=0A+ return RTE_PTR= _ADD(last->addr, last->len);=0A+}=0A+=0A+=0A /* map a particular resource from= a file */=0A void *=0A pci_map_resource(void *requested_addr, int fd, off_t o= ffset, size_t size)=0Adiff --git a/lib/librte_eal/linuxapp/eal/eal_pci_uio.c b= /lib/librte_eal/linuxapp/eal/eal_pci_uio.c=0Aindex 7e62266..5090bf1 100644=0A-= -- a/lib/librte_eal/linuxapp/eal/eal_pci_uio.c=0A+++ b/lib/librte_eal/linuxapp= /eal/eal_pci_uio.c=0A@@ -48,6 +48,8 @@=0A =0A static int pci_parse_sysfs_value= (const char *filename, uint64_t *val);=0A =0A+void *pci_map_addr =3D NULL;=0A+= =0A =0A #define OFF_MAX ((uint64_t)(off_t)-1)=0A static int=0A@@ = -371,10 +373,15 @@ pci_uio_map_resource(struct rte_pci_device *dev)=0A if (= maps[j].addr !=3D NULL)=0A fail =3D 1;=0A else {=0A- mapaddr =3D pci= _map_resource(NULL, fd, (off_t)offset,=0A+ if (pci_map_addr =3D=3D NULL)=0A= + pci_map_addr =3D pci_find_max_end_va();=0A+=0A+ mapaddr =3D pci_map_r= esource(pci_map_addr, fd, (off_t)offset,=0A (size_t)maps[j].size);=0A = if (mapaddr =3D=3D NULL)=0A fail =3D 1;=0A+=0A+ pci_map_addr =3D RTE= _PTR_ADD(pci_map_addr, maps[j].size);=0A }=0A =0A if (fail) {=0Adiff --g= it a/lib/librte_eal/linuxapp/eal/eal_pci_vfio.c b/lib/librte_eal/linuxapp/eal/= eal_pci_vfio.c=0Aindex c776ddc..fb6ee7a 100644=0A--- a/lib/librte_eal/linuxapp= /eal/eal_pci_vfio.c=0A+++ b/lib/librte_eal/linuxapp/eal/eal_pci_vfio.c=0A@@ -7= 20,8 +720,17 @@ pci_vfio_map_resource(struct rte_pci_device *dev)=0A if (i =3D= =3D msix_bar)=0A continue;=0A =0A- bar_addr =3D pci_map_resource(maps[i].a= ddr, vfio_dev_fd, reg.offset,=0A- reg.size);=0A+ if (internal_config.proce= ss_type =3D=3D RTE_PROC_PRIMARY) {=0A+ if (pci_map_addr =3D=3D NULL)=0A+ = pci_map_addr =3D pci_find_max_end_va();=0A+=0A+ bar_addr =3D pci_map_resourc= e(pci_map_addr, vfio_dev_fd, reg.offset,=0A+ reg.size);=0A+ pci_map_addr= =3D RTE_PTR_ADD(pci_map_addr, reg.size);=0A+ } else {=0A+ bar_addr =3D pci= _map_resource(maps[i].addr, vfio_dev_fd, reg.offset,=0A+ reg.size);=0A+ }= =0A =0A if (bar_addr =3D=3D NULL) {=0A RTE_LOG(ERR, EAL, " %s mapping BA= R%i failed: %s\n", pci_addr, i,=0Adiff --git a/lib/librte_eal/linuxapp/eal/inc= lude/eal_pci_init.h b/lib/librte_eal/linuxapp/eal/include/eal_pci_init.h=0Aind= ex d758bee..1070eb8 100644=0A--- a/lib/librte_eal/linuxapp/eal/include/eal_pci= _init.h=0A+++ b/lib/librte_eal/linuxapp/eal/include/eal_pci_init.h=0A@@ -59,6 = +59,12 @@ struct mapped_pci_resource {=0A TAILQ_HEAD(mapped_pci_res_list, mapp= ed_pci_resource);=0A extern struct mapped_pci_res_list *pci_res_list;=0A =0A+/= *=0A+ * Helper function to map PCI resources right after hugepages in virtual = memory=0A+ */=0A+extern void *pci_map_addr;=0A+void *pci_find_max_end_va(void)= ;=0A+=0A void *pci_map_resource(void *requested_addr, int fd, off_t offset,=0A= size_t size);=0A =0A-- =0A1.8.1.4