Attempt to modularize mmaps/allocation. Freeing still not working correctly

Signed-off-by: Ilias Apalodimas <ilias.apalodimas@linaro.org>

7 files changed, 131 insertions, 119 deletions

diff --git a/Makefile b/Makefile
index 118be80..65cee0e 100644
--- a/Makefile
+++ b/Makefile
@@ -1,8 +1,12 @@
 CC=gcc
 CFLAGS=-Iinclude/ -Wall -Werror -Wunused
 
-all: drivers/r8169.o api/vfio_api.o
-	#$(CC) drivers/r8169.c -o r8169 $(CFLAGS)
-	$(CC) drivers/r8169.o api/vfio_api.o -o r8169 $(CFLAGS)
+
+all: r8169
+
+R8169_OBJS:= drivers/r8169.o api/vfio_api.o api/mm.o
+r8169: $(R8169_OBJS)
+	$(CC) $(R8169_OBJS) -o r8169 $(CFLAGS)
+
 clean:
 	rm -f r8169 && rm `find -name *.o` -f
diff --git a/api/mm.c b/api/mm.c
new file mode 100644
index 0000000..019ebad
--- /dev/null
+++ b/api/mm.c
@@ -0,0 +1,70 @@
+#include <stdio.h>
+#include <errno.h>
+#include <linux/types.h>
+#include <linux/vfio.h>
+#include <sys/mman.h>
+#include <sys/ioctl.h>
+
+#include <mm.h>
+
+#define TO_GB(x) (x * 1024ULL * 1024ULL * 1024ULL)
+
+void *iomem_init(void)
+{
+	void *tmp;
+	/*
+	 * reserve a 4GB contiguous address space and position it,
+	 * no pages are actually allocated and mapped into this address space
+	 * it is just making sure that overtime, we'll have 4GB contiguous
+	 */
+	tmp = mmap(NULL, TO_GB(4), PROT_READ | PROT_WRITE, MAP_SHARED |
+		   MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
+	if (tmp == MAP_FAILED) {
+		printf("Could not reserve a contiguous address space\n");
+		return NULL;
+	}
+
+	return tmp;
+}
+
+int iomem_alloc(int device, unsigned int size, void **iomem_current,
+		struct iomem *iomem)
+{
+	void *tmp;
+	int ret;
+	__u64 location;
+	struct vfio_iommu_type1_dma_map dma_map = { .argsz = sizeof(dma_map) };
+
+	if (size >= 32 * 1024 * 1024)
+		return -EINVAL;
+	if ((size & 0xFFF) != 0)
+		return -EINVAL; /* size should be a 4K aligned quantity */
+
+	/* get a portion of the 4GB window created at init time */
+	tmp = mmap(*iomem_current, size, PROT_READ | PROT_WRITE,
+		   MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE | MAP_FIXED, -1,
+		   0);
+	if (tmp == MAP_FAILED)
+		return -ENOMEM;
+
+	location = (__u64)tmp;
+	*iomem_current += size;
+
+	iomem->vaddr = location;
+	iomem->size = size;
+
+	dma_map.vaddr = iomem->vaddr;
+	dma_map.size = iomem->size;
+
+	/* kernel driver fills dma_map.iova with the proper allocated IOVA */
+	ret = ioctl(device, VFIO_IOMMU_MAP_DMA, &dma_map);
+	if (ret != 0)
+		return -ENOMEM;
+	iomem->iova = dma_map.iova;
+
+	printf("iomem_alloc: VA(%llx) -> physmem(%dKB) <- IOVA(%llx)\n",
+	       iomem->vaddr, size/1024, iomem->iova);
+
+	return 0;
+}
+
diff --git a/drivers/r8169.c b/drivers/r8169.c
index f77a6e3..af8a454 100644
--- a/drivers/r8169.c
+++ b/drivers/r8169.c
@@ -4,16 +4,19 @@
 #include <malloc.h>
 #include <string.h>
 #include <arpa/inet.h>
-#include <linux/vfio.h>
-#include <sys/ioctl.h>
-#include <sys/mman.h>
 #include <endian.h>
 #include <stdlib.h>
-//#include <linux/types.h>
+#include <errno.h>
+
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+
+#include <linux/vfio.h>
 
 /* Us */
 #include <drivers/r8169.h>
 #include <vfio_api.h>
+#include <mm.h>
 
 typedef unsigned long dma_addr_t;
 char *rxBuffers[256];
@@ -32,84 +35,6 @@ static void print_packet(unsigned char *buffer)
 	}
 }
 
-#define IOMEM_CHUNKS 4096
-iomem iomem_array[IOMEM_CHUNKS];
-int iomem_count;
-__u64 iomem_base;
-__u64 iomem_current;
-
-int iomem_init(void)
-{
-	void *tmp;
-	iomem_count = 0;
-	memset(iomem_array, 0, sizeof(iomem_array));
-	iomem_base = 1 * 1024ULL * 1024ULL * 1024ULL * 1024ULL;
-
-	iomem_current = iomem_base;
-	/* reserve a 4GB contiguous address space and position it, if possible at 8GB */
-	/* no pages are actually allocated and mapped into this address space */
-	/* it is just making sure that overtime, we'll have 4GB contiguous */
-	tmp = mmap((void *)iomem_base, 4 * 1024ULL * 1024ULL * 1024ULL,
-		   PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS |
-		   MAP_NORESERVE, -1, 0);
-	if (tmp == MAP_FAILED) {
-		printf("Could not reserve a contiguous 4GB address space\n");
-		return -1;
-	}
-	iomem_base = (__u64)tmp;
-	iomem_current = iomem_base;
-
-	return 0;
-}
-
-struct iomem *iomem_alloc(int device, unsigned int size)
-{
-	void *tmp;
-	int ret;
-	__u64 location;
-	struct vfio_iommu_type1_dma_map dma_map = { .argsz = sizeof(dma_map) };
-
-	if (size >= 32 * 1024 * 1024)
-		return NULL;
-	if ((size & 0xFFF) != 0)
-		return NULL; /* size should be a 4K aligned quantity */
-	if (iomem_count >= IOMEM_CHUNKS)
-		return NULL;
-
-	/* get a portion of the 4GB window created at init time */
-	tmp = mmap((void *)iomem_current, size, PROT_READ | PROT_WRITE,
-		   MAP_SHARED | MAP_ANONYMOUS | MAP_NORESERVE | MAP_FIXED, -1,
-		   0);
-	if (tmp == MAP_FAILED)
-		return NULL;
-	if (iomem_base == 0) {
-		iomem_base = (__u64)tmp;
-		iomem_current = iomem_base + size;
-		location = iomem_base;
-	} else {
-		location = iomem_current;
-		iomem_current += size;
-	}
-
-	iomem_array[iomem_count].vaddr = location;
-	iomem_array[iomem_count].size = size;
-
-	dma_map.vaddr = iomem_array[iomem_count].vaddr;
-	dma_map.size = iomem_array[iomem_count].size;
-
-	/* kernel driver fills dma_map.iova with the proper allocated IOVA */
-	ret = ioctl(device, VFIO_IOMMU_MAP_DMA, &dma_map);
-	if (ret != 0)
-		return NULL;
-	iomem_array[iomem_count].iova = dma_map.iova;
-
-	printf("iomem_alloc: VA(%llx) -> physmem(%dKB) <- IOVA(%llx)\n",
-	       iomem_array[iomem_count].vaddr, size/1024,
-	       iomem_array[iomem_count].iova);
-
-	return &iomem_array[iomem_count++];
-}
-
 static inline void rtl8169_mark_to_asic(struct RxDesc *desc, __u32 rx_buf_sz)
 {
 	__u32 eor = le32_to_cpu(desc->opts1) & RingEnd;
@@ -133,21 +58,15 @@ static inline void rtl8169_mark_as_last_descriptor(struct RxDesc *desc)
 	desc->opts1 |= cpu_to_le32(RingEnd);
 }
 
-int rtl8169_rx_fill(int device, struct RxDesc *rxRing)
+int rtl8169_rx_fill(int device, struct RxDesc *rxRing, struct iomem *data)
 {
 	int i;
-	struct iomem *packetArea;
-
-	packetArea = iomem_alloc(device, 2 * 1024 * 1024);
-	if (packetArea == NULL)
-		return -1;
 
 	for (i = 0; i < NUM_RX_DESC; i++) {
-		rtl8169_map_to_asic(&rxRing[i], packetArea->iova + i * 2048,
-				    2048);
-		rxBuffers[i] = (char *)(packetArea->vaddr + i * 2048);
+		rtl8169_map_to_asic(&rxRing[i], data->iova + i * 2048, 2048);
+		rxBuffers[i] = (char *)(data->vaddr + i * 2048);
 	}
-	rtl8169_mark_as_last_descriptor(&rxRing[255]);
+	rtl8169_mark_as_last_descriptor(&rxRing[NUM_RX_DESC - 1]);
 
 	return 0;
 }
@@ -159,24 +78,29 @@ void usage(char *name)
 
 int main(int argc, char *argv[])
 {
-	int container, group, device, i = 0;
+	int container = -1, group = -1, device, i = 0;
 	int group_id;
-	char group_uuid[128];
+	char group_uuid[64]; /* 37 should be enough */
 	struct vfio_group_status group_status = { .argsz = sizeof(group_status) };
 	struct vfio_iommu_type1_info iommu_info = { .argsz = sizeof(iommu_info) };
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
 	struct vfio_region_info region_info = { .argsz = sizeof(region_info) };
-	struct RxDesc *rxRing;
+	struct RxDesc *rxRing = NULL;
+	void *iobase, *iocur;
+	struct iomem rx_data;
+	int ret;
 
 	if (argc != 3) {
 		usage(argv[0]);
 		return -1;
 	}
-	iomem_init();
+	iobase = iomem_init();
+	if (!iobase)
+		return -ENOMEM;
+	iocur = iobase;
 
 	/* Create a new container */
 	container = get_container();
-
 	if (container < 0)
 		goto out;
 
@@ -196,12 +120,16 @@ int main(int argc, char *argv[])
 		      MAP_SHARED, device, region_info.offset);
 	if (rxRing == MAP_FAILED) {
 		printf("Could not reserve on contiguous 4GB address space\n");
-		return -1;
+		goto out;
 	}
 
-	if (rtl8169_rx_fill(device, rxRing) != 0) {
+	ret = iomem_alloc(device, 2 * 1024 * 1024, &iocur, &rx_data);
+	if (ret)
+		goto out;
+
+	if (rtl8169_rx_fill(device, rxRing, &rx_data) != 0) {
 		printf("Could not fill ring\n");
-		return -1;
+		goto out;
 	}
 
 	/* signal ready */
@@ -246,11 +174,10 @@ int main(int argc, char *argv[])
 					/* && 	(dev->features & NETIF_F_RXALL) */
 					)
 					goto process_pkt;
-			}
-			else {
+			} else {
 				//dma_addr_t addr;
 				int pkt_size;
-			process_pkt:
+process_pkt:
 				//addr = le64_to_cpu(rxRing[i].addr);
 				if (1) // likely(!(dev->features & NETIF_F_RXFCS)))
 					pkt_size = (status & 0x00003fff) - 4;
@@ -296,7 +223,7 @@ int main(int argc, char *argv[])
 				print_packet((unsigned char *)rxBuffers[i]);
 				printf("\n");
 			}
-		/*release_descriptor: */
+			/* release_descriptor: */
 			rxRing[i].opts2 = 0;
 			rtl8169_mark_to_asic(&rxRing[i], 2048);
 		}
@@ -304,8 +231,12 @@ int main(int argc, char *argv[])
 	}
 
 out:
-	close(group);
-	close(container);
+	if (rxRing)
+		munmap(rxRing, region_info.size);
+	if (group)
+		close(group);
+	if (container)
+		close(container);
 	return -1;
 
 }
diff --git a/include/drivers/r8169.h b/include/drivers/r8169.h
index 1b125f2..d3aca73 100644
--- a/include/drivers/r8169.h
+++ b/include/drivers/r8169.h
@@ -17,12 +17,6 @@
 #define R8169_TX_RING_BYTES     (NUM_TX_DESC * sizeof(struct TxDesc))
 #define R8169_RX_RING_BYTES     (NUM_RX_DESC * sizeof(struct RxDesc))
 
-typedef struct iomem {
-	__u64 vaddr;
-	__u64 iova;
-	__u64 size;
-} iomem;
-
 /* drivers/ethernet/realtek/r8169.c */
 enum rtl_register_content {
 	/* InterruptStatusBits */
diff --git a/include/mm.h b/include/mm.h
new file mode 100644
index 0000000..c89fd5c
--- /dev/null
+++ b/include/mm.h
@@ -0,0 +1,13 @@
+#ifndef _MM_H
+#define _MM_H
+#define IOMEM_CHUNKS 4096
+struct iomem {
+	__u64 vaddr;
+	__u64 iova;
+	__u64 size;
+};
+
+void *iomem_init(void);
+int iomem_alloc(int device, unsigned int size, void **iomem_curent,
+		struct iomem *iomem);
+#endif
diff --git a/include/vfio_api.h b/include/vfio_api.h
index 7485e67..72a368a 100644
--- a/include/vfio_api.h
+++ b/include/vfio_api.h
@@ -1,5 +1,5 @@
-#ifndef VFIO_API_H
-#define VFIO_API_H
+#ifndef _VFIO_API_H
+#define _VFIO_API_H
 int dma_map_type1(int fd, unsigned long sz, void **vaddr, uint64_t iova);
 int dma_unmap_type1(int fd, unsigned long sz, void *vaddr, uint64_t iova);
 int get_group(int grp_id);
diff --git a/patches/linux-r8169.patch b/patches/linux-r8169.patch
index 4daf6f3..2d4a3a4 100644
--- a/patches/linux-r8169.patch
+++ b/patches/linux-r8169.patch
@@ -80,8 +80,8 @@ index 8f1623b..25181fc 100644
 +	u64 iova;
 +	void *vaddr;
 +	struct device *dev;
-+	u32 size:25; /* maximum of 32MB */
-+	u32 direction:2; /*  DMA_FROM_DEVICE... */
++	u32 size; /* maximum of 32MB */
++	enum dma_data_direction direction; /*  DMA_FROM_DEVICE... */
 +};
 +
 +typedef struct netmdev {