All of lore.kernel.org
 help / color / mirror / Atom feed
* [RFC PATCH 06/25] staging: erofs: add directory operations
From: Gao Xiang @ 2018-07-24  2:36 UTC (permalink / raw)

In-Reply-To: <1532399805-65674-1-git-send-email-gaoxiang25@huawei.com>

This adds functions for directory, mainly readdir.

Signed-off-by: Miao Xie <miaoxie at huawei.com>
Signed-off-by: Chao Yu <yuchao0 at huawei.com>
Signed-off-by: Gao Xiang <gaoxiang25 at huawei.com>
---
 drivers/staging/erofs/dir.c | 145 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 145 insertions(+)
 create mode 100644 drivers/staging/erofs/dir.c

diff --git a/drivers/staging/erofs/dir.c b/drivers/staging/erofs/dir.c
new file mode 100644
index 0000000..d910424
--- /dev/null
+++ b/drivers/staging/erofs/dir.c
@@ -0,0 +1,145 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * linux/drivers/staging/erofs/dir.c
+ *
+ * Copyright (C) 2017-2018 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Gao Xiang <gaoxiang25 at huawei.com>
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file COPYING in the main directory of the Linux
+ * distribution for more details.
+ */
+#include "internal.h"
+
+const unsigned char erofs_filetype_table[EROFS_FT_MAX] = {
+	[EROFS_FT_UNKNOWN]	= DT_UNKNOWN,
+	[EROFS_FT_REG_FILE]	= DT_REG,
+	[EROFS_FT_DIR]		= DT_DIR,
+	[EROFS_FT_CHRDEV]	= DT_CHR,
+	[EROFS_FT_BLKDEV]	= DT_BLK,
+	[EROFS_FT_FIFO]		= DT_FIFO,
+	[EROFS_FT_SOCK]		= DT_SOCK,
+	[EROFS_FT_SYMLINK]	= DT_LNK,
+};
+
+static int erofs_fill_dentries(struct dir_context *ctx,
+	void *dentry_blk, unsigned *ofs,
+	unsigned nameoff, unsigned maxsize)
+{
+	struct erofs_dirent *de = dentry_blk;
+	const struct erofs_dirent *end = dentry_blk + nameoff;
+
+	de = dentry_blk + *ofs;
+	while (de < end) {
+		const char *de_name;
+		int de_namelen;
+		unsigned char d_type;
+#ifdef CONFIG_EROFS_FS_DEBUG
+		unsigned dbg_namelen;
+		unsigned char dbg_namebuf[EROFS_NAME_LEN];
+#endif
+
+		if (unlikely(de->file_type < EROFS_FT_MAX))
+			d_type = erofs_filetype_table[de->file_type];
+		else
+			d_type = DT_UNKNOWN;
+
+		nameoff = le16_to_cpu(de->nameoff);
+		de_name = (char *)dentry_blk + nameoff;
+
+		de_namelen = unlikely(de + 1 >= end) ?
+			/* last directory entry */
+			strnlen(de_name, maxsize - nameoff) :
+			le16_to_cpu(de[1].nameoff) - nameoff;
+
+		/* the corrupted directory found */
+		BUG_ON(de_namelen < 0);
+
+#ifdef CONFIG_EROFS_FS_DEBUG
+		dbg_namelen = min(EROFS_NAME_LEN - 1, de_namelen);
+		memcpy(dbg_namebuf, de_name, dbg_namelen);
+		dbg_namebuf[dbg_namelen] = '\0';
+
+		debugln("%s, found de_name %s de_len %d d_type %d", __func__,
+			dbg_namebuf, de_namelen, d_type);
+#endif
+
+		if (!dir_emit(ctx, de_name, de_namelen,
+					le64_to_cpu(de->nid), d_type))
+			/* stoped by some reason */
+			return 1;
+		++de;
+		*ofs += sizeof(struct erofs_dirent);
+	}
+	*ofs = maxsize;
+	return 0;
+}
+
+static int erofs_readdir(struct file *f, struct dir_context *ctx)
+{
+	struct inode *dir = file_inode(f);
+	struct address_space *mapping = dir->i_mapping;
+	const size_t dirsize = i_size_read(dir);
+	unsigned i = ctx->pos / EROFS_BLKSIZ;
+	unsigned ofs = ctx->pos % EROFS_BLKSIZ;
+	int err = 0;
+	bool initial = true;
+
+	while (ctx->pos < dirsize) {
+		struct page *dentry_page;
+		struct erofs_dirent *de;
+		unsigned nameoff, maxsize;
+
+		dentry_page = read_mapping_page(mapping, i, NULL);
+		if (IS_ERR(dentry_page))
+			continue;
+
+		lock_page(dentry_page);
+		de = (struct erofs_dirent *)kmap(dentry_page);
+
+		nameoff = le16_to_cpu(de->nameoff);
+
+		if (unlikely(nameoff < sizeof(struct erofs_dirent) ||
+			nameoff >= PAGE_SIZE)) {
+			errln("%s, invalid de[0].nameoff %u",
+				__func__, nameoff);
+
+			err = -EIO;
+			goto skip_this;
+		}
+
+		maxsize = min_t(unsigned, dirsize - ctx->pos + ofs, PAGE_SIZE);
+
+		/* search dirents at the arbitrary position */
+		if (unlikely(initial)) {
+			initial = false;
+
+			ofs = roundup(ofs, sizeof(struct erofs_dirent));
+			if (unlikely(ofs >= nameoff))
+				goto skip_this;
+		}
+
+		err = erofs_fill_dentries(ctx, de, &ofs, nameoff, maxsize);
+skip_this:
+		kunmap(dentry_page);
+
+		unlock_page(dentry_page);
+		put_page(dentry_page);
+
+		ctx->pos = blknr_to_addr(i) + ofs;
+
+		if (unlikely(err))
+			break;
+		++i;
+		ofs = 0;
+	}
+	return err < 0 ? err : 0;
+}
+
+const struct file_operations erofs_dir_fops = {
+	.llseek		= generic_file_llseek,
+	.read		= generic_read_dir,
+	.iterate	= erofs_readdir,
+};
+
-- 
1.9.1

^ permalink raw reply related

* [RFC PATCH 05/25] staging: erofs: add inode operations
From: Gao Xiang @ 2018-07-24  2:36 UTC (permalink / raw)

In-Reply-To: <1532399805-65674-1-git-send-email-gaoxiang25@huawei.com>

This adds core functions to get, read an inode.

Signed-off-by: Miao Xie <miaoxie at huawei.com>
Signed-off-by: Chao Yu <yuchao0 at huawei.com>
Signed-off-by: Gao Xiang <gaoxiang25 at huawei.com>
---
 drivers/staging/erofs/inode.c | 231 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 231 insertions(+)
 create mode 100644 drivers/staging/erofs/inode.c

diff --git a/drivers/staging/erofs/inode.c b/drivers/staging/erofs/inode.c
new file mode 100644
index 0000000..b9a9d2c
--- /dev/null
+++ b/drivers/staging/erofs/inode.c
@@ -0,0 +1,231 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * linux/drivers/staging/erofs/inode.c
+ *
+ * Copyright (C) 2017-2018 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Gao Xiang <gaoxiang25 at huawei.com>
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file COPYING in the main directory of the Linux
+ * distribution for more details.
+ */
+#include "internal.h"
+
+/* no locking */
+static int read_inode(struct inode *inode, void *data)
+{
+	struct erofs_vnode *vi = EROFS_V(inode);
+	struct erofs_inode_v1 *v1 = data;
+	const unsigned advise = le16_to_cpu(v1->i_advise);
+
+	vi->data_mapping_mode = __inode_data_mapping(advise);
+
+	if (unlikely(vi->data_mapping_mode >= EROFS_INODE_LAYOUT_MAX)) {
+		errln("unknown data mapping mode %u of nid %llu",
+			vi->data_mapping_mode, vi->nid);
+		DBG_BUGON(1);
+		return -EIO;
+	}
+
+	if (__inode_version(advise) == EROFS_INODE_LAYOUT_V2) {
+		struct erofs_inode_v2 *v2 = data;
+
+		vi->inode_isize = sizeof(struct erofs_inode_v2);
+		vi->xattr_isize = ondisk_xattr_ibody_size(v2->i_xattr_icount);
+
+		vi->raw_blkaddr = le32_to_cpu(v2->i_u.raw_blkaddr);
+		inode->i_mode = le16_to_cpu(v2->i_mode);
+
+		i_uid_write(inode, le32_to_cpu(v2->i_uid));
+		i_gid_write(inode, le32_to_cpu(v2->i_gid));
+		set_nlink(inode, le32_to_cpu(v2->i_nlink));
+
+		/* ns timestamp */
+		inode->i_mtime.tv_sec = inode->i_ctime.tv_sec =
+			le64_to_cpu(v2->i_ctime);
+		inode->i_mtime.tv_nsec = inode->i_ctime.tv_nsec =
+			le32_to_cpu(v2->i_ctime_nsec);
+
+		inode->i_size = le64_to_cpu(v2->i_size);
+	} else if (__inode_version(advise) == EROFS_INODE_LAYOUT_V1) {
+		struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb);
+
+		vi->inode_isize = sizeof(struct erofs_inode_v1);
+		vi->xattr_isize = ondisk_xattr_ibody_size(v1->i_xattr_icount);
+
+		vi->raw_blkaddr = le32_to_cpu(v1->i_u.raw_blkaddr);
+		inode->i_mode = le16_to_cpu(v1->i_mode);
+
+		i_uid_write(inode, le16_to_cpu(v1->i_uid));
+		i_gid_write(inode, le16_to_cpu(v1->i_gid));
+		set_nlink(inode, le16_to_cpu(v1->i_nlink));
+
+		/* use build time to derive all file time */
+		inode->i_mtime.tv_sec = inode->i_ctime.tv_sec =
+			sbi->build_time;
+		inode->i_mtime.tv_nsec = inode->i_ctime.tv_nsec =
+			sbi->build_time_nsec;
+
+		inode->i_size = le32_to_cpu(v1->i_size);
+	} else {
+		errln("unsupported on-disk inode version %u of nid %llu",
+			__inode_version(advise), vi->nid);
+		DBG_BUGON(1);
+		return -EIO;
+	}
+
+	/* measure inode.i_blocks as the generic filesystem */
+	inode->i_blocks = ((inode->i_size - 1) >> 9) + 1;
+	return 0;
+}
+
+/*
+ * try_lock can be required since locking order is:
+ *   file data(fs_inode)
+ *        meta(bd_inode)
+ * but the majority of the callers is "iget",
+ * in that case we are pretty sure no deadlock since
+ * no data operations exist. However I tend to
+ * try_lock since it takes no much overhead and
+ * will success immediately.
+ */
+int fill_inline_data(struct inode *inode, void *data, unsigned m_pofs)
+{
+	struct erofs_vnode *vi = EROFS_V(inode);
+	int mode = vi->data_mapping_mode;
+
+	DBG_BUGON(mode >= EROFS_INODE_LAYOUT_MAX);
+
+	/* should be inode inline C */
+	if (mode != EROFS_INODE_LAYOUT_INLINE)
+		return 0;
+
+	/* fast symlink (following ext4) */
+	if (S_ISLNK(inode->i_mode) && inode->i_size < PAGE_SIZE) {
+		char *lnk = kmalloc(inode->i_size + 1, GFP_KERNEL);
+
+		if (unlikely(lnk == NULL))
+			return -ENOMEM;
+
+		m_pofs += vi->inode_isize + vi->xattr_isize;
+		BUG_ON(m_pofs + inode->i_size > PAGE_SIZE);
+
+		/* get in-page inline data */
+		memcpy(lnk, data + m_pofs, inode->i_size);
+		lnk[inode->i_size] = '\0';
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0))
+		vi->i_link = lnk;
+#else
+		inode->i_link = lnk;
+#endif
+		set_inode_fast_symlink(inode);
+	}
+	return -EAGAIN;
+}
+
+int fill_inode(struct inode *inode, int isdir)
+{
+	struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb);
+	struct erofs_vnode *vi = EROFS_V(inode);
+	struct page *page;
+	void *data;
+	int err;
+	erofs_blk_t blkaddr;
+	unsigned ofs;
+
+	blkaddr = erofs_blknr(iloc(sbi, vi->nid));
+	ofs = erofs_blkoff(iloc(sbi, vi->nid));
+
+	debugln("%s, reading inode nid %llu at %u of blkaddr %u",
+		__func__, vi->nid, ofs, blkaddr);
+
+	page = erofs_get_meta_page(inode->i_sb, blkaddr, isdir);
+
+	if (IS_ERR(page)) {
+		errln("failed to get inode (nid: %llu) page, err %ld",
+			vi->nid, PTR_ERR(page));
+		return PTR_ERR(page);
+	}
+
+	BUG_ON(!PageUptodate(page));
+	data = page_address(page);
+
+	err = read_inode(inode, data + ofs);
+	if (!err) {
+		/* setup the new inode */
+		if (S_ISREG(inode->i_mode)) {
+			inode->i_fop = &generic_ro_fops;
+		} else if (S_ISDIR(inode->i_mode)) {
+			inode->i_op =
+				&erofs_dir_iops;
+			inode->i_fop = &erofs_dir_fops;
+		} else if (S_ISLNK(inode->i_mode)) {
+			/* by default, page_get_link is used for symlink */
+			inode->i_op =
+				&page_symlink_inode_operations;
+			inode_nohighmem(inode);
+		} else {
+			err = -EIO;
+			goto out_unlock;
+		}
+
+		if (is_inode_layout_compression(inode)) {
+			err = -ENOTSUPP;
+			goto out_unlock;
+		}
+
+		inode->i_mapping->a_ops = &erofs_raw_access_aops;
+
+		/* fill last page if inline data is available */
+		fill_inline_data(inode, data, ofs);
+	}
+
+out_unlock:
+	unlock_page(page);
+	put_page(page);
+	return err;
+}
+
+struct inode *erofs_iget(struct super_block *sb,
+	erofs_nid_t nid, bool isdir)
+{
+	struct inode *inode = iget_locked(sb, nid);
+
+	if (unlikely(inode == NULL))
+		return ERR_PTR(-ENOMEM);
+
+	if (inode->i_state & I_NEW) {
+		int err;
+		struct erofs_vnode *vi = EROFS_V(inode);
+		vi->nid = nid;
+
+		err = fill_inode(inode, isdir);
+		if (likely(!err))
+			unlock_new_inode(inode);
+		else {
+			iget_failed(inode);
+			inode = ERR_PTR(err);
+		}
+	}
+	return inode;
+}
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0))
+#include <linux/namei.h>
+
+static void *erofs_follow_fast_link(struct dentry *dentry, struct nameidata *nd)
+{
+	struct erofs_vnode *vi = EROFS_V(d_inode(dentry));
+
+	nd_set_link(nd, (char *)vi->i_link);
+	return NULL;
+}
+
+const struct inode_operations simple_symlink_inode_operations = {
+	.follow_link = erofs_follow_fast_link,
+	.readlink = generic_readlink
+};
+#endif
+
-- 
1.9.1

^ permalink raw reply related

* [RFC PATCH 04/25] staging: erofs: add raw address_space operations
From: Gao Xiang @ 2018-07-24  2:36 UTC (permalink / raw)

In-Reply-To: <1532399805-65674-1-git-send-email-gaoxiang25@huawei.com>

This commit adds functions for meta and raw data, and also
provides address_space_operations for raw data access.

Signed-off-by: Miao Xie <miaoxie at huawei.com>
Signed-off-by: Chao Yu <yuchao0 at huawei.com>
Signed-off-by: Gao Xiang <gaoxiang25 at huawei.com>
---
 drivers/staging/erofs/data.c | 377 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 377 insertions(+)
 create mode 100644 drivers/staging/erofs/data.c

diff --git a/drivers/staging/erofs/data.c b/drivers/staging/erofs/data.c
new file mode 100644
index 0000000..29703b9
--- /dev/null
+++ b/drivers/staging/erofs/data.c
@@ -0,0 +1,377 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * linux/drivers/staging/erofs/data.c
+ *
+ * Copyright (C) 2017-2018 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Gao Xiang <gaoxiang25 at huawei.com>
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file COPYING in the main directory of the Linux
+ * distribution for more details.
+ */
+#include "internal.h"
+#include <linux/prefetch.h>
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0))
+static inline void read_endio(struct bio *bio, int err)
+#else
+static inline void read_endio(struct bio *bio)
+#endif
+{
+	int i;
+	struct bio_vec *bvec;
+#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0))
+	const int err = bio->bi_status;
+#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 3, 0))
+	const int err = bio->bi_error;
+#endif
+
+	bio_for_each_segment_all(bvec, bio, i) {
+		struct page *page = bvec->bv_page;
+
+		/* page is already locked */
+		BUG_ON(PageUptodate(page));
+
+		if (unlikely(err))
+			SetPageError(page);
+		else
+			SetPageUptodate(page);
+
+		unlock_page(page);
+		/* page could be reclaimed now */
+	}
+	bio_put(bio);
+}
+
+static void __submit_bio(struct bio *bio, unsigned op, unsigned op_flags)
+{
+	bio_set_op_attrs(bio, op, op_flags);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0))
+	submit_bio(0, bio);
+#else
+	submit_bio(bio);
+#endif
+}
+
+static struct bio *prepare_bio(struct super_block *sb,
+	erofs_blk_t blkaddr, unsigned nr_pages)
+{
+	struct bio *bio = bio_alloc(GFP_NOIO | __GFP_NOFAIL, nr_pages);
+
+	BUG_ON(bio == NULL);
+
+	bio->bi_end_io = read_endio;
+	bio_set_dev(bio, sb->s_bdev);
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0))
+	bio->bi_sector = blkaddr << LOG_SECTORS_PER_BLOCK;
+#else
+	bio->bi_iter.bi_sector = blkaddr << LOG_SECTORS_PER_BLOCK;
+#endif
+	return bio;
+}
+
+/* prio -- true is used for dir */
+struct page *erofs_get_meta_page(struct super_block *sb,
+	erofs_blk_t blkaddr, bool prio)
+{
+	struct inode *bd_inode = sb->s_bdev->bd_inode;
+	struct address_space *mapping = bd_inode->i_mapping;
+	struct page *page;
+
+repeat:
+	page = find_or_create_page(mapping, blkaddr,
+	/*
+	 * Prefer looping in the allocator rather than here,
+	 * at least that code knows what it's doing.
+	 */
+		mapping_gfp_constraint(mapping, ~__GFP_FS) | __GFP_NOFAIL);
+
+	BUG_ON(!page || !PageLocked(page));
+
+	if (!PageUptodate(page)) {
+		struct bio *bio;
+		int err;
+
+		bio = prepare_bio(sb, blkaddr, 1);
+		err = bio_add_page(bio, page, PAGE_SIZE, 0);
+		BUG_ON(err != PAGE_SIZE);
+
+		__submit_bio(bio, REQ_OP_READ,
+			REQ_META | (prio ? REQ_PRIO : 0));
+
+		lock_page(page);
+
+		/* the page has been truncated by others? */
+		if (unlikely(page->mapping != mapping)) {
+			unlock_page(page);
+			put_page(page);
+			goto repeat;
+		}
+
+		/* more likely a read error */
+		if (unlikely(!PageUptodate(page))) {
+			unlock_page(page);
+			put_page(page);
+
+			page = ERR_PTR(-EIO);
+		}
+	}
+	return page;
+}
+
+static int erofs_map_blocks_flatmode(struct inode *inode,
+	struct erofs_map_blocks *map,
+	int flags)
+{
+	erofs_blk_t nblocks, lastblk;
+	u64 offset = map->m_la;
+	struct erofs_vnode *vi = EROFS_V(inode);
+
+	BUG_ON(is_inode_layout_compression(inode));
+
+	nblocks = DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
+	lastblk = nblocks - is_inode_layout_inline(inode);
+
+	if (unlikely(offset >= inode->i_size)) {
+		/* leave out-of-bound access unmapped */
+		map->m_flags = 0;
+		map->m_plen = 0;
+		goto out;
+	}
+
+	/* there is no hole in flatmode */
+	map->m_flags = EROFS_MAP_MAPPED;
+
+	if (offset < blknr_to_addr(lastblk)) {
+		map->m_pa = blknr_to_addr(vi->raw_blkaddr) + map->m_la;
+		map->m_plen = blknr_to_addr(lastblk) - offset;
+	} else if (is_inode_layout_inline(inode)) {
+		/* 2 - inode inline B: inode, [xattrs], inline last blk... */
+		struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb);
+
+		map->m_pa = iloc(sbi, vi->nid) + vi->inode_isize +
+			vi->xattr_isize + erofs_blkoff(map->m_la);
+		map->m_plen = inode->i_size - offset;
+
+		/* inline data should locate in one meta block */
+		BUG_ON(erofs_blkoff(map->m_pa) + map->m_plen > PAGE_SIZE);
+		map->m_flags |= EROFS_MAP_META;
+	} else {
+		errln("internal error @ nid: %llu (size %llu), m_la 0x%llx",
+			vi->nid, inode->i_size, map->m_la);
+		BUG();
+	}
+
+out:
+	map->m_llen = map->m_plen;
+	debugln("%s, m_la 0x%llx m_pa %llx m_len %llu",
+		__func__, map->m_la, map->m_pa, map->m_plen);
+	return 0;
+}
+
+int erofs_map_blocks(struct inode *inode,
+	struct erofs_map_blocks *map, int flags)
+{
+	if (unlikely(is_inode_layout_compression(inode)))
+		return -ENOTSUPP;
+
+	return erofs_map_blocks_flatmode(inode, map, flags);
+}
+
+static inline struct bio *erofs_read_raw_page(
+	struct bio *bio,
+	struct address_space *mapping,
+	struct page *page,
+	erofs_off_t *last_block,
+	unsigned nblocks,
+	bool ra)
+{
+	struct inode *inode = mapping->host;
+	erofs_off_t current_block = (erofs_off_t)page->index;
+	int err;
+
+	BUG_ON(!nblocks);
+
+	if (PageUptodate(page)) {
+		err = 0;
+		goto has_updated;
+	}
+
+	if (cleancache_get_page(page) == 0) {
+		err = 0;
+		SetPageUptodate(page);
+		goto has_updated;
+	}
+
+	/* note that for readpage case, bio also equals to NULL */
+	if (bio != NULL &&
+		/* not continuous */
+		*last_block + 1 != current_block) {
+submit_bio_retry:
+		__submit_bio(bio, REQ_OP_READ, 0);
+		bio = NULL;
+	}
+
+	if (bio == NULL) {
+		struct erofs_map_blocks map = {
+			.m_la = blknr_to_addr(current_block),
+		};
+
+		err = erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW);
+		if (unlikely(err))
+			goto err_out;
+
+		/* zero out the holed page */
+		if (unlikely(!(map.m_flags & EROFS_MAP_MAPPED))) {
+			zero_user_segment(page, 0, PAGE_SIZE);
+			SetPageUptodate(page);
+
+			/* imply err = 0, see erofs_map_blocks */
+			goto has_updated;
+		}
+
+		/* for RAW access mode, m_plen must be equal to m_llen */
+		BUG_ON(map.m_plen != map.m_llen);
+
+		/* deal with inline page */
+		if (map.m_flags & EROFS_MAP_META) {
+			void *vsrc, *vto;
+			struct page *ipage;
+
+			BUG_ON(map.m_plen > PAGE_SIZE);
+
+			ipage = erofs_get_meta_page(inode->i_sb,
+				erofs_blknr(map.m_pa), 0);
+
+			if (IS_ERR(ipage)) {
+				err = PTR_ERR(ipage);
+				goto err_out;
+			}
+
+			vsrc = kmap_atomic(ipage);
+			vto = kmap_atomic(page);
+			memcpy(vto, vsrc + erofs_blkoff(map.m_pa), map.m_plen);
+			memset(vto + map.m_plen, 0, PAGE_SIZE - map.m_plen);
+			kunmap_atomic(vto);
+			kunmap_atomic(vsrc);
+			flush_dcache_page(page);
+
+			SetPageUptodate(page);
+			/* TODO: could we unlock the page earlier? */
+			unlock_page(ipage);
+			put_page(ipage);
+
+			/* imply err = 0, see erofs_map_blocks */
+			goto has_updated;
+		}
+
+		/* pa must be block-aligned for raw reading */
+		BUG_ON(erofs_blkoff(map.m_pa) != 0);
+
+		/* max # of continuous pages */
+		if (nblocks > DIV_ROUND_UP(map.m_plen, PAGE_SIZE))
+			nblocks = DIV_ROUND_UP(map.m_plen, PAGE_SIZE);
+		if (nblocks > BIO_MAX_PAGES)
+			nblocks = BIO_MAX_PAGES;
+
+		bio = prepare_bio(inode->i_sb, erofs_blknr(map.m_pa), nblocks);
+	}
+
+	err = bio_add_page(bio, page, PAGE_SIZE, 0);
+	/* out of the extent or bio is full */
+	if (err < PAGE_SIZE)
+		goto submit_bio_retry;
+
+	*last_block = current_block;
+
+	/* shift in advance in case of it followed by too many gaps */
+	if (unlikely(bio->bi_vcnt >= bio->bi_max_vecs)) {
+		/* err should reassign to 0 after submitting */
+		err = 0;
+		goto submit_bio_out;
+	}
+
+	return bio;
+
+err_out:
+	/* for sync reading, set page error immediately */
+	if (!ra) {
+		SetPageError(page);
+		ClearPageUptodate(page);
+	}
+has_updated:
+	unlock_page(page);
+
+	/* if updated manually, continuous pages has a gap */
+	if (bio != NULL)
+submit_bio_out:
+		__submit_bio(bio, REQ_OP_READ, 0);
+
+	return unlikely(err) ? ERR_PTR(err) : NULL;
+}
+
+/*
+ * since we dont have write or truncate flows, so no inode
+ * locking needs to be held at the moment.
+ */
+static int erofs_raw_access_readpage(struct file *file, struct page *page)
+{
+	erofs_off_t last_block;
+	struct bio *bio;
+
+	bio = erofs_read_raw_page(NULL, page->mapping,
+		page, &last_block, 1, false);
+
+	if (IS_ERR(bio))
+		return PTR_ERR(bio);
+
+	BUG_ON(bio != NULL);	/* since we have only one bio -- must be NULL */
+	return 0;
+}
+
+static int erofs_raw_access_readpages(struct file *filp,
+	struct address_space *mapping,
+	struct list_head *pages, unsigned nr_pages)
+{
+	erofs_off_t last_block;
+	struct bio *bio = NULL;
+	gfp_t gfp = readahead_gfp_mask(mapping);
+
+	for (; nr_pages; --nr_pages) {
+		struct page *page = list_entry(pages->prev, struct page, lru);
+
+		prefetchw(&page->flags);
+		list_del(&page->lru);
+
+		if (!add_to_page_cache_lru(page, mapping, page->index, gfp)) {
+			bio = erofs_read_raw_page(bio, mapping, page,
+				&last_block, nr_pages, true);
+
+			/* all the page errors are ignored when readahead */
+			if (IS_ERR(bio)) {
+				pr_err("%s, readahead error at page %lu of nid %llu\n",
+					__func__, page->index,
+					EROFS_V(mapping->host)->nid);
+
+				bio = NULL;
+			}
+		}
+
+		/* pages could still be locked */
+		page_cache_release(page);
+	}
+	BUG_ON(!list_empty(pages));
+
+	/* the rare case (end in gaps) */
+	if (unlikely(bio != NULL))
+		__submit_bio(bio, REQ_OP_READ, 0);
+	return 0;
+}
+
+/* for uncompressed (aligned) files and raw access for other files */
+const struct address_space_operations erofs_raw_access_aops = {
+	.readpage = erofs_raw_access_readpage,
+	.readpages = erofs_raw_access_readpages,
+};
+
-- 
1.9.1

^ permalink raw reply related

* [RFC PATCH 03/25] staging: erofs: add super block operations
From: Gao Xiang @ 2018-07-24  2:36 UTC (permalink / raw)

In-Reply-To: <1532399805-65674-1-git-send-email-gaoxiang25@huawei.com>

This commit adds erofs super block operations, including (u)mount,
remount_fs, show_options, statfs, in addition to some private
icache management functions.

Signed-off-by: Miao Xie <miaoxie at huawei.com>
Signed-off-by: Chao Yu <yuchao0 at huawei.com>
Signed-off-by: Gao Xiang <gaoxiang25 at huawei.com>
---
 drivers/staging/erofs/super.c | 423 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 423 insertions(+)
 create mode 100644 drivers/staging/erofs/super.c

diff --git a/drivers/staging/erofs/super.c b/drivers/staging/erofs/super.c
new file mode 100644
index 0000000..31bfef0
--- /dev/null
+++ b/drivers/staging/erofs/super.c
@@ -0,0 +1,423 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * linux/drivers/staging/erofs/super.c
+ *
+ * Copyright (C) 2017-2018 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Gao Xiang <gaoxiang25 at huawei.com>
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file COPYING in the main directory of the Linux
+ * distribution for more details.
+ */
+#include <linux/module.h>
+#include <linux/buffer_head.h>
+#include <linux/statfs.h>
+#include <linux/parser.h>
+#include "internal.h"
+
+static struct kmem_cache *erofs_inode_cachep __read_mostly;
+
+static void init_once(void *ptr)
+{
+	struct erofs_vnode *vi = ptr;
+
+	inode_init_once(&vi->vfs_inode);
+}
+
+static int erofs_init_inode_cache(void)
+{
+	erofs_inode_cachep = kmem_cache_create("erofs_inode",
+		sizeof(struct erofs_vnode), 0,
+		SLAB_RECLAIM_ACCOUNT, init_once);
+
+	return erofs_inode_cachep != NULL ? 0 : -ENOMEM;
+}
+
+static void erofs_exit_inode_cache(void)
+{
+	BUG_ON(erofs_inode_cachep == NULL);
+	kmem_cache_destroy(erofs_inode_cachep);
+}
+
+static struct inode *alloc_inode(struct super_block *sb)
+{
+	struct erofs_vnode *vi =
+		kmem_cache_alloc(erofs_inode_cachep, GFP_KERNEL);
+
+	if (vi == NULL)
+		return NULL;
+
+	/* zero out everything except vfs_inode */
+	memset(vi, 0, offsetof(struct erofs_vnode, vfs_inode));
+	return &vi->vfs_inode;
+}
+
+static void i_callback(struct rcu_head *head)
+{
+	struct inode *inode = container_of(head, struct inode, i_rcu);
+	struct erofs_vnode *vi = EROFS_V(inode);
+
+	/* be careful RCU symlink path (see ext4_inode_info->i_data)! */
+	if (is_inode_fast_symlink(inode))
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0))
+		kfree(vi->i_link);
+#else
+		kfree(inode->i_link);
+#endif
+
+	kfree(vi->xattr_shared_xattrs);
+
+	kmem_cache_free(erofs_inode_cachep, vi);
+}
+
+static void destroy_inode(struct inode *inode)
+{
+	call_rcu(&inode->i_rcu, i_callback);
+}
+
+static int superblock_read(struct super_block *sb)
+{
+	struct erofs_sb_info *sbi;
+	struct buffer_head *bh;
+	struct erofs_super_block *layout;
+	unsigned blkszbits;
+	int ret;
+
+	bh = sb_bread(sb, 0);
+
+	if (bh == NULL) {
+		errln("cannot read erofs superblock");
+		return -EIO;
+	}
+
+	sbi = EROFS_SB(sb);
+	layout = (struct erofs_super_block *)((u8 *)bh->b_data
+		 + EROFS_SUPER_OFFSET);
+
+	ret = -EINVAL;
+	if (le32_to_cpu(layout->magic) != EROFS_SUPER_MAGIC_V1) {
+		errln("cannot find valid erofs superblock");
+		goto out;
+	}
+
+	blkszbits = layout->blkszbits;
+	/* 9(512 bytes) + LOG_SECTORS_PER_BLOCK == LOG_BLOCK_SIZE */
+	if (unlikely(blkszbits != LOG_BLOCK_SIZE)) {
+		errln("blksize %u isn't supported on this platform",
+			1 << blkszbits);
+		goto out;
+	}
+
+	sbi->blocks = le32_to_cpu(layout->blocks);
+	sbi->meta_blkaddr = le32_to_cpu(layout->meta_blkaddr);
+	sbi->islotbits = ffs(sizeof(struct erofs_inode_v1)) - 1;
+
+	sbi->root_nid = le16_to_cpu(layout->root_nid);
+	sbi->inos = le64_to_cpu(layout->inos);
+
+	sbi->build_time = le64_to_cpu(layout->build_time);
+	sbi->build_time_nsec = le32_to_cpu(layout->build_time_nsec);
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0))
+	memcpy(sb->s_uuid, layout->uuid, sizeof(layout->uuid));
+#else
+	memcpy(&sb->s_uuid, layout->uuid, sizeof(layout->uuid));
+#endif
+	memcpy(sbi->volume_name, layout->volume_name,
+		sizeof(layout->volume_name));
+
+	ret = 0;
+out:
+	brelse(bh);
+	return ret;
+}
+
+static void default_options(struct erofs_sb_info *sbi)
+{
+}
+
+enum {
+	Opt_err
+};
+
+static match_table_t erofs_tokens = {
+	{Opt_err, NULL}
+};
+
+static int parse_options(struct super_block *sb, char *options)
+{
+	substring_t args[MAX_OPT_ARGS];
+	char *p;
+
+	if (!options)
+		return 0;
+
+	while ((p = strsep(&options, ",")) != NULL) {
+		int token;
+
+		if (!*p)
+			continue;
+
+		args[0].to = args[0].from = NULL;
+		token = match_token(p, erofs_tokens, args);
+
+		switch (token) {
+		default:
+			errln("Unrecognized mount option \"%s\" "
+					"or missing value", p);
+			return -EINVAL;
+		}
+	}
+	return 0;
+}
+
+static int erofs_read_super(struct super_block *sb,
+	const char *dev_name, void *data, int silent)
+{
+	struct inode *inode;
+	struct erofs_sb_info *sbi;
+	int err = -EINVAL;
+
+	infoln("read_super, device -> %s", dev_name);
+	infoln("options -> %s", (char *)data);
+
+	if (unlikely(!sb_set_blocksize(sb, EROFS_BLKSIZ))) {
+		errln("failed to set erofs blksize");
+		goto err;
+	}
+
+	sbi = kzalloc(sizeof(struct erofs_sb_info), GFP_KERNEL);
+	if (unlikely(sbi == NULL)) {
+		err = -ENOMEM;
+		goto err;
+	}
+	sb->s_fs_info = sbi;
+
+	err = superblock_read(sb);
+	if (err)
+		goto err_sbread;
+
+	sb->s_magic = EROFS_SUPER_MAGIC;
+	sb->s_flags |= MS_RDONLY | MS_NOATIME;
+	sb->s_maxbytes = MAX_LFS_FILESIZE;
+	sb->s_time_gran = 1;
+
+	sb->s_op = &erofs_sops;
+
+	/* set erofs default mount options */
+	default_options(sbi);
+
+	err = parse_options(sb, data);
+	if (err)
+		goto err_parseopt;
+
+	if (!silent)
+		infoln("root inode @ nid %llu", ROOT_NID(sbi));
+
+	/* get the root inode */
+	inode = erofs_iget(sb, ROOT_NID(sbi), true);
+	if (IS_ERR(inode)) {
+		err = PTR_ERR(inode);
+		goto err_iget;
+	}
+
+	if (!S_ISDIR(inode->i_mode)) {
+		errln("rootino(nid %llu) is not a directory(i_mode %o)",
+			ROOT_NID(sbi), inode->i_mode);
+		err = -EINVAL;
+		goto err_isdir;
+	}
+
+	sb->s_root = d_make_root(inode);
+	if (sb->s_root == NULL) {
+		err = -ENOMEM;
+		goto err_makeroot;
+	}
+
+	/* save the device name to sbi */
+	sbi->dev_name = __getname();
+	if (sbi->dev_name == NULL) {
+		err = -ENOMEM;
+		goto err_devname;
+	}
+
+	snprintf(sbi->dev_name, PATH_MAX, "%s", dev_name);
+	sbi->dev_name[PATH_MAX - 1] = '\0';
+
+	/*
+	 * We already have a positive dentry, which was instantiated
+	 * by d_make_root. Just need to d_rehash it.
+	 */
+	d_rehash(sb->s_root);
+
+	if (!silent)
+		infoln("mounted on %s with opts: %s.", dev_name,
+			(char *)data);
+	return 0;
+	/*
+	 * please add a label for each exit point and use
+	 * the following name convention, thus new features
+	 * can be integrated easily without renaming labels.
+	 */
+err_devname:
+	dput(sb->s_root);
+err_makeroot:
+err_isdir:
+	if (sb->s_root == NULL)
+		iput(inode);
+err_iget:
+err_parseopt:
+err_sbread:
+	sb->s_fs_info = NULL;
+	kfree(sbi);
+err:
+	return err;
+}
+
+/*
+ * could be triggered after deactivate_locked_super()
+ * is called, thus including umount and failed to initialize.
+ */
+static void erofs_put_super(struct super_block *sb)
+{
+	struct erofs_sb_info *sbi = EROFS_SB(sb);
+
+	/* for cases which are failed in "read_super" */
+	if (sbi == NULL)
+		return;
+
+	WARN_ON(sb->s_magic != EROFS_SUPER_MAGIC);
+
+	infoln("unmounted for %s", sbi->dev_name);
+	__putname(sbi->dev_name);
+
+	kfree(sbi);
+	sb->s_fs_info = NULL;
+}
+
+
+struct erofs_mount_private {
+	const char *dev_name;
+	char *options;
+};
+
+/* support mount_bdev() with options */
+static int erofs_fill_super(struct super_block *sb,
+	void *_priv, int silent)
+{
+	struct erofs_mount_private *priv = _priv;
+
+	return erofs_read_super(sb, priv->dev_name,
+		priv->options, silent);
+}
+
+static struct dentry *erofs_mount(
+	struct file_system_type *fs_type, int flags,
+	const char *dev_name, void *data)
+{
+	struct erofs_mount_private priv = {
+		.dev_name = dev_name,
+		.options = data
+	};
+
+	return mount_bdev(fs_type, flags, dev_name,
+		&priv, erofs_fill_super);
+}
+
+static void erofs_kill_sb(struct super_block *sb)
+{
+	kill_block_super(sb);
+}
+
+static struct file_system_type erofs_fs_type = {
+	.owner          = THIS_MODULE,
+	.name           = "erofs",
+	.mount          = erofs_mount,
+	.kill_sb        = erofs_kill_sb,
+	.fs_flags       = FS_REQUIRES_DEV,
+};
+MODULE_ALIAS_FS("erofs");
+
+int __init erofs_module_init(void)
+{
+	int err;
+
+	erofs_check_ondisk_layout_definitions();
+	infoln("initializing erofs " EROFS_VERSION);
+
+	err = erofs_init_inode_cache();
+	if (err)
+		goto icache_err;
+
+	err = register_filesystem(&erofs_fs_type);
+	if (err)
+		goto fs_err;
+
+	infoln("successfully to initialize erofs");
+	return 0;
+
+fs_err:
+	erofs_exit_inode_cache();
+icache_err:
+	return err;
+}
+
+void __exit erofs_module_exit(void)
+{
+	unregister_filesystem(&erofs_fs_type);
+	erofs_exit_inode_cache();
+	infoln("successfully finalize erofs");
+}
+
+/* get filesystem statistics */
+static int erofs_statfs(struct dentry *dentry, struct kstatfs *buf)
+{
+	struct super_block *sb = dentry->d_sb;
+	struct erofs_sb_info *sbi = EROFS_SB(sb);
+	u64 id = huge_encode_dev(sb->s_bdev->bd_dev);
+
+	buf->f_type = sb->s_magic;
+	buf->f_bsize = EROFS_BLKSIZ;
+	buf->f_blocks = sbi->blocks;
+	buf->f_bfree = buf->f_bavail = 0;
+
+	buf->f_files = ULLONG_MAX;
+	buf->f_ffree = ULLONG_MAX - sbi->inos;
+
+	buf->f_namelen = EROFS_NAME_LEN;
+
+	buf->f_fsid.val[0] = (u32)id;
+	buf->f_fsid.val[1] = (u32)(id >> 32);
+	return 0;
+}
+
+static int erofs_show_options(struct seq_file *seq, struct dentry *root)
+{
+	return 0;
+}
+
+static int erofs_remount(struct super_block *sb, int *flags, char *data)
+{
+	BUG_ON(!sb_rdonly(sb));
+
+	*flags |= MS_RDONLY;
+	return 0;
+}
+
+const struct super_operations erofs_sops = {
+	.put_super = erofs_put_super,
+	.alloc_inode = alloc_inode,
+	.destroy_inode = destroy_inode,
+	.statfs = erofs_statfs,
+	.show_options = erofs_show_options,
+	.remount_fs = erofs_remount,
+};
+
+module_init(erofs_module_init);
+module_exit(erofs_module_exit);
+
+MODULE_DESCRIPTION("Enhanced ROM File System");
+MODULE_AUTHOR("Gao Xiang, Yu Chao, Miao Xie, CONSUMER BG, HUAWEI Inc.");
+MODULE_LICENSE("GPL");
+
-- 
1.9.1

^ permalink raw reply related

* [RFC PATCH 02/25] staging: erofs: add erofs in-memory stuffs
From: Gao Xiang @ 2018-07-24  2:36 UTC (permalink / raw)

In-Reply-To: <1532399805-65674-1-git-send-email-gaoxiang25@huawei.com>

 - erofs_sb_info:
   contains erofs-specific in-memory information.

 - erofs_vnode:
   contains vfs_inode and other fs-specific information.
   same as super block, the only one in-memory definition exists.

 - erofs_map_blocks
   plays a role in the file L2P mapping

Signed-off-by: Miao Xie <miaoxie at huawei.com>
Signed-off-by: Chao Yu <yuchao0 at huawei.com>
Signed-off-by: Gao Xiang <gaoxiang25 at huawei.com>
---
 drivers/staging/erofs/internal.h | 286 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 286 insertions(+)
 create mode 100644 drivers/staging/erofs/internal.h

diff --git a/drivers/staging/erofs/internal.h b/drivers/staging/erofs/internal.h
new file mode 100644
index 0000000..5056177
--- /dev/null
+++ b/drivers/staging/erofs/internal.h
@@ -0,0 +1,286 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * linux/drivers/staging/erofs/internal.h
+ *
+ * Copyright (C) 2017-2018 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Gao Xiang <gaoxiang25 at huawei.com>
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file COPYING in the main directory of the Linux
+ * distribution for more details.
+ */
+#ifndef __INTERNAL_H
+#define __INTERNAL_H
+
+#include <linux/fs.h>
+#include <linux/dcache.h>
+#include <linux/mm.h>
+#include <linux/pagemap.h>
+#include <linux/bio.h>
+#include <linux/buffer_head.h>
+#include <linux/cleancache.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include "erofs_fs.h"
+
+#include "staging.h"
+
+/* redefine pr_fmt "erofs: " */
+#undef pr_fmt
+#define pr_fmt(fmt) "erofs: " fmt
+
+#define errln(x, ...)   pr_err(x "\n", ##__VA_ARGS__)
+#define infoln(x, ...)  pr_info(x "\n", ##__VA_ARGS__)
+#ifdef CONFIG_EROFS_FS_DEBUG
+#define debugln(x, ...) pr_debug(x "\n", ##__VA_ARGS__)
+
+#define dbg_might_sleep         might_sleep
+#define DBG_BUGON               BUG_ON
+#else
+#define debugln(x, ...)         ((void)0)
+
+#define dbg_might_sleep()       ((void)0)
+#define DBG_BUGON(...)          ((void)0)
+#endif
+
+/* EROFS_SUPER_MAGIC_V1 to represent the whole file system */
+#define EROFS_SUPER_MAGIC   EROFS_SUPER_MAGIC_V1
+
+typedef u64 erofs_nid_t;
+
+struct erofs_sb_info {
+	u32 blocks;
+	u32 meta_blkaddr;
+
+	/* inode slot unit size in bit shift */
+	unsigned char islotbits;
+
+	u32 build_time_nsec;
+	u64 build_time;
+
+	/* what we really care is nid, rather than ino.. */
+	erofs_nid_t root_nid;
+	/* used for statfs, f_files - f_favail */
+	u64 inos;
+
+	u8 uuid[16];                    /* 128-bit uuid for volume */
+	u8 volume_name[16];             /* volume name */
+	char *dev_name;
+
+	unsigned int mount_opt;
+};
+
+#define EROFS_SB(sb) ((struct erofs_sb_info *)(sb)->s_fs_info)
+#define EROFS_I_SB(inode) ((struct erofs_sb_info *)(inode)->i_sb->s_fs_info)
+
+#define clear_opt(sbi, option)	((sbi)->mount_opt &= ~EROFS_MOUNT_##option)
+#define set_opt(sbi, option)	((sbi)->mount_opt |= EROFS_MOUNT_##option)
+#define test_opt(sbi, option)	((sbi)->mount_opt & EROFS_MOUNT_##option)
+
+/* we strictly follow PAGE_SIZE and no buffer head yet */
+#define LOG_BLOCK_SIZE		PAGE_SHIFT
+
+#undef LOG_SECTORS_PER_BLOCK
+#define LOG_SECTORS_PER_BLOCK	(PAGE_SHIFT - 9)
+
+#undef SECTORS_PER_BLOCK
+#define SECTORS_PER_BLOCK	(1 << SECTORS_PER_BLOCK)
+
+#define EROFS_BLKSIZ		(1 << LOG_BLOCK_SIZE)
+
+#if (EROFS_BLKSIZ % 4096 || !EROFS_BLKSIZ)
+#error erofs cannot be used in this platform
+#endif
+
+#define ROOT_NID(sb)		((sb)->root_nid)
+
+typedef u64 erofs_off_t;
+
+/* data type for filesystem-wide blocks number */
+typedef u32 erofs_blk_t;
+
+#define erofs_blknr(addr)       ((addr) / EROFS_BLKSIZ)
+#define erofs_blkoff(addr)      ((addr) % EROFS_BLKSIZ)
+#define blknr_to_addr(nr)       ((erofs_off_t)(nr) * EROFS_BLKSIZ)
+
+static inline erofs_off_t iloc(struct erofs_sb_info *sbi, erofs_nid_t nid)
+{
+	return blknr_to_addr(sbi->meta_blkaddr) + (nid << sbi->islotbits);
+}
+
+#define inode_set_inited_xattr(inode)   (EROFS_V(inode)->flags |= 1)
+#define inode_has_inited_xattr(inode)   (EROFS_V(inode)->flags & 1)
+
+struct erofs_vnode {
+	erofs_nid_t nid;
+	unsigned int flags;
+
+	unsigned char data_mapping_mode;
+	/* inline size in bytes */
+	unsigned char inode_isize;
+	unsigned short xattr_isize;
+
+#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0))
+	char *i_link;
+#endif
+	unsigned xattr_shared_count;
+	unsigned *xattr_shared_xattrs;
+
+	erofs_blk_t raw_blkaddr;
+
+	/* the corresponding vfs inode */
+	struct inode vfs_inode;
+};
+
+#define EROFS_V(ptr)	\
+	container_of(ptr, struct erofs_vnode, vfs_inode)
+
+#define __inode_advise(x, bit, bits) \
+	(((x) >> (bit)) & ((1 << (bits)) - 1))
+
+#define __inode_version(advise)	\
+	__inode_advise(advise, EROFS_I_VERSION_BIT,	\
+		EROFS_I_VERSION_BITS)
+
+#define __inode_data_mapping(advise)	\
+	__inode_advise(advise, EROFS_I_DATA_MAPPING_BIT,\
+		EROFS_I_DATA_MAPPING_BITS)
+
+static inline unsigned long inode_datablocks(struct inode *inode)
+{
+	/* since i_size cannot be changed */
+	return DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ);
+}
+
+static inline bool is_inode_layout_plain(struct inode *inode)
+{
+	return EROFS_V(inode)->data_mapping_mode == EROFS_INODE_LAYOUT_PLAIN;
+}
+
+static inline bool is_inode_layout_compression(struct inode *inode)
+{
+	return EROFS_V(inode)->data_mapping_mode ==
+					EROFS_INODE_LAYOUT_COMPRESSION;
+}
+
+static inline bool is_inode_layout_inline(struct inode *inode)
+{
+	return EROFS_V(inode)->data_mapping_mode == EROFS_INODE_LAYOUT_INLINE;
+}
+
+extern const struct super_operations erofs_sops;
+extern const struct inode_operations erofs_dir_iops;
+extern const struct file_operations erofs_dir_fops;
+
+extern const struct address_space_operations erofs_raw_access_aops;
+
+/*
+ * Logical to physical block mapping, used by erofs_map_blocks()
+ *
+ * Different with other file systems, it is used for 2 access modes:
+ *
+ * 1) RAW access mode:
+ *
+ * Users pass a valid (m_lblk, m_lofs -- usually 0) pair,
+ * and get the valid m_pblk, m_pofs and the longest m_len(in bytes).
+ *
+ * Note that m_lblk in the RAW access mode refers to the number of
+ * the compressed ondisk block rather than the uncompressed
+ * in-memory block for the compressed file.
+ *
+ * m_pofs equals to m_lofs except for the inline data page.
+ *
+ * 2) Normal access mode:
+ *
+ * If the inode is not compressed, it has no difference with
+ * the RAW access mode. However, if the inode is compressed,
+ * users should pass a valid (m_lblk, m_lofs) pair, and get
+ * the needed m_pblk, m_pofs, m_len to get the compressed data
+ * and the updated m_lblk, m_lofs which indicates the start
+ * of the corresponding uncompressed data in the file.
+ */
+enum {
+	BH_Zipped = BH_PrivateStart,
+};
+
+/* Has a disk mapping */
+#define EROFS_MAP_MAPPED	(1 << BH_Mapped)
+/* Located in metadata (could be copied from bd_inode) */
+#define EROFS_MAP_META		(1 << BH_Meta)
+/* The extent has been compressed */
+#define EROFS_MAP_ZIPPED	(1 << BH_Zipped)
+
+struct erofs_map_blocks {
+	erofs_off_t m_pa, m_la;
+	u64 m_plen, m_llen;
+
+	unsigned int m_flags;
+};
+
+/* Flags used by erofs_map_blocks() */
+#define EROFS_GET_BLOCKS_RAW    0x0001
+
+/* data.c */
+extern struct page *erofs_get_meta_page(struct super_block *sb,
+	erofs_blk_t blkaddr, bool prio);
+extern int erofs_map_blocks(struct inode *, struct erofs_map_blocks *, int);
+
+static inline struct page *erofs_get_inline_page(struct inode *inode,
+	erofs_blk_t blkaddr)
+{
+	return erofs_get_meta_page(inode->i_sb,
+		blkaddr, S_ISDIR(inode->i_mode));
+}
+
+/* inode.c */
+extern struct inode *erofs_iget(struct super_block *sb,
+	erofs_nid_t nid, bool dir);
+
+/* dir.c */
+int erofs_namei(struct inode *dir, struct qstr *name,
+	erofs_nid_t *nid, unsigned *d_type);
+
+/* xattr.c */
+extern const struct xattr_handler *erofs_xattr_handlers[];
+
+/* symlink */
+static inline void set_inode_fast_symlink(struct inode *inode)
+{
+	inode->i_op = &simple_symlink_inode_operations;
+}
+
+static inline bool is_inode_fast_symlink(struct inode *inode)
+{
+	return inode->i_op == &simple_symlink_inode_operations;
+}
+
+static inline void *erofs_vmap(struct page **pages, unsigned int count)
+{
+#ifdef CONFIG_EROFS_FS_USE_VM_MAP_RAM
+	int i = 0;
+
+	while (1) {
+		void *addr = vm_map_ram(pages, count, -1, PAGE_KERNEL);
+		/* retry two more times (totally 3 times) */
+		if (addr != NULL || ++i >= 3)
+			return addr;
+		vm_unmap_aliases();
+	}
+	return NULL;
+#else
+	return vmap(pages, count, VM_MAP, PAGE_KERNEL);
+#endif
+}
+
+static inline void erofs_vunmap(const void *mem, unsigned int count)
+{
+#ifdef CONFIG_EROFS_FS_USE_VM_MAP_RAM
+	vm_unmap_ram(mem, count);
+#else
+	vunmap(mem);
+#endif
+}
+
+#endif
+
-- 
1.9.1

^ permalink raw reply related

* [RFC PATCH 01/25] staging: erofs: add on-disk layout
From: Gao Xiang @ 2018-07-24  2:36 UTC (permalink / raw)

In-Reply-To: <1532399805-65674-1-git-send-email-gaoxiang25@huawei.com>

This commit adds the on-disk layout header file of erofs.

Note that the on-disk layout is still WIP, and some fields are
reserved for the future use by design.

Any comments are welcome.

Thanks-to: Li Guifu <liguifu2 at huawei.com>
Thanks-to: Sun Qiuyang <sunqiuyang at huawei.com>
Signed-off-by: Miao Xie <miaoxie at huawei.com>
Signed-off-by: Chao Yu <yuchao0 at huawei.com>
Signed-off-by: Gao Xiang <gaoxiang25 at huawei.com>
---
 drivers/staging/erofs/erofs_fs.h | 271 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 271 insertions(+)
 create mode 100644 drivers/staging/erofs/erofs_fs.h

diff --git a/drivers/staging/erofs/erofs_fs.h b/drivers/staging/erofs/erofs_fs.h
new file mode 100644
index 0000000..2594635
--- /dev/null
+++ b/drivers/staging/erofs/erofs_fs.h
@@ -0,0 +1,271 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Apache-2.0
+ *
+ * linux/drivers/staging/erofs/erofs_fs.h
+ *
+ * Copyright (C) 2017-2018 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Gao Xiang <gaoxiang25 at huawei.com>
+ *
+ * This file is dual-licensed; you may select either the GNU General Public
+ * License version 2 or Apache License, Version 2.0. See the file COPYING
+ * in the main directory of the Linux distribution for more details.
+ */
+#ifndef __EROFS_FS_H
+#define __EROFS_FS_H
+
+/* Enhanced(Extended) ROM File System */
+#define EROFS_SUPER_MAGIC_V1    0xE0F5E1E2
+#define EROFS_SUPER_OFFSET      1024
+
+struct erofs_super_block {
+/*  0 */__le32 magic;           /* in the little endian */
+/*  4 */__le32 checksum;        /* crc32c(super_block) */
+/*  8 */__le32 features;
+/* 12 */__u8 blkszbits;         /* support block_size == PAGE_SIZE only */
+/* 13 */__u8 reserved;
+
+/* 14 */__le16 root_nid;
+/* 16 */__le64 inos;            /* total valid ino # (== f_files - f_favail) */
+
+/* 24 */__le64 build_time;      /* inode v1 time derivation */
+/* 32 */__le32 build_time_nsec;
+/* 36 */__le32 blocks;          /* used for statfs */
+/* 40 */__le32 meta_blkaddr;
+/* 44 */__le32 xattr_blkaddr;
+/* 48 */__u8 uuid[16];          /* 128-bit uuid for volume */
+/* 64 */__u8 volume_name[16];   /* volume name */
+
+/* 80 */__u8 reserved2[48];     /* 128 bytes */
+} __packed;
+
+#define __EROFS_BIT(_prefix, _cur, _pre)	enum {	\
+	_prefix ## _cur ## _BIT = _prefix ## _pre ## _BIT + \
+		_prefix ## _pre ## _BITS }
+
+/*
+ * erofs inode data mapping:
+ * 0 - inode plain without inline data A:
+ * inode, [xattrs], ... | ... | no-holed data
+ * 1 - inode VLE compression B:
+ * inode, [xattrs], extents ... | ...
+ * 2 - inode plain with inline data C:
+ * inode, [xattrs], last_inline_data, ... | ... | no-holed data
+ * 3~7 - reserved
+ */
+enum {
+	EROFS_INODE_LAYOUT_PLAIN,
+	EROFS_INODE_LAYOUT_COMPRESSION,
+	EROFS_INODE_LAYOUT_INLINE,
+	EROFS_INODE_LAYOUT_MAX
+};
+#define EROFS_I_VERSION_BITS            1
+#define EROFS_I_DATA_MAPPING_BITS       3
+
+#define EROFS_I_VERSION_BIT             0
+__EROFS_BIT(EROFS_I_, DATA_MAPPING, VERSION);
+
+struct erofs_inode_v1 {
+/*  0 */__le16 i_advise;
+
+/* 1 header + n-1 * 4 bytes inline xattr to keep continuity */
+/*  2 */__le16 i_xattr_icount;
+/*  4 */__le16 i_mode;
+/*  6 */__le16 i_nlink;
+/*  8 */__le32 i_size;
+/* 12 */__le32 i_reserved;
+/* 16 */union {
+		/* file total compressed blocks for data mapping 1 */
+		__le32 compressed_blocks;
+		__le32 raw_blkaddr;
+
+		/* for device files, used to indicate old/new device # */
+		__le32 rdev;
+	} i_u __packed;
+/* 20 */__le32 i_ino;           /* only used for 32-bit stat compatibility */
+/* 24 */__le16 i_uid;
+/* 26 */__le16 i_gid;
+/* 28 */__le32 i_checksum;
+} __packed;
+
+/* 32 bytes on-disk inode */
+#define EROFS_INODE_LAYOUT_V1   0
+/* 64 bytes on-disk inode */
+#define EROFS_INODE_LAYOUT_V2   1
+
+struct erofs_inode_v2 {
+	__le16 i_advise;
+
+	/* 1 header + n-1 * 4 bytes inline xattr to keep continuity */
+	__le16 i_xattr_icount;
+	__le16 i_mode;
+	__le16 i_reserved;      /* 8 bytes */
+	__le64 i_size;          /* 16 bytes */
+	union {
+		/* file total compressed blocks for data mapping 1 */
+		__le32 compressed_blocks;
+		__le32 raw_blkaddr;
+
+		/* for device files, used to indicate old/new device # */
+		__le32 rdev;
+	} i_u __packed;
+
+	/* only used for 32-bit stat compatibility */
+	__le32 i_ino;           /* 24 bytes */
+
+	__le32 i_uid;
+	__le32 i_gid;
+	__le64 i_ctime;         /* 32 bytes */
+	__le32 i_ctime_nsec;
+	__le32 i_nlink;
+	__u8   i_reserved2[12];
+	__le32 i_checksum;      /* 64 bytes */
+} __packed;
+
+#define EROFS_MAX_SHARED_XATTRS         (128)
+/* h_shared_count between 129 ... 255 are special # */
+#define EROFS_SHARED_XATTR_EXTENT       (255)
+
+/*
+ * inline xattrs (n == i_xattr_icount):
+ * erofs_xattr_ibody_header(1) + (n - 1) * 4 bytes
+ *          12 bytes           /                   \
+ *                            /                     \
+ *                           /-----------------------\
+ *                           |  erofs_xattr_entries+ |
+ *                           +-----------------------+
+ * inline xattrs must starts in erofs_xattr_ibody_header,
+ * for read-only fs, no need to introduce h_refcount
+ */
+struct erofs_xattr_ibody_header {
+	__le32 h_checksum;
+	__u8   h_shared_count;
+	__u8   h_reserved[7];
+	__le32 h_shared_xattrs[0];      /* shared xattr id array */
+} __packed;
+
+/* Name indexes */
+#define EROFS_XATTR_INDEX_USER              1
+#define EROFS_XATTR_INDEX_POSIX_ACL_ACCESS  2
+#define EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT 3
+#define EROFS_XATTR_INDEX_TRUSTED           4
+#define EROFS_XATTR_INDEX_LUSTRE            5
+#define EROFS_XATTR_INDEX_SECURITY          6
+
+/* xattr entry (for both inline & shared xattrs) */
+struct erofs_xattr_entry {
+	__u8   e_name_len;      /* length of name */
+	__u8   e_name_index;    /* attribute name index */
+	__le16 e_value_size;    /* size of attribute value */
+	/* followed by e_name and e_value */
+	char   e_name[0];       /* attribute name */
+} __packed;
+
+#define ondisk_xattr_ibody_size(count)	({\
+	u32 __count = le16_to_cpu(count); \
+	((__count) == 0) ? 0 : \
+	sizeof(struct erofs_xattr_ibody_header) + \
+		sizeof(__u32) * ((__count) - 1); })
+
+#define EROFS_XATTR_ALIGN(size) round_up(size, sizeof(struct erofs_xattr_entry))
+#define EROFS_XATTR_ENTRY_SIZE(entry) EROFS_XATTR_ALIGN( \
+	sizeof(struct erofs_xattr_entry) + \
+	(entry)->e_name_len + le16_to_cpu((entry)->e_value_size))
+
+/* have to be aligned with 8 bytes on disk */
+struct erofs_extent_header {
+	__le32 eh_checksum;
+	__le32 eh_reserved[3];
+} __packed;
+
+/*
+ * Z_EROFS Variable-sized Logical Extent cluster type:
+ *    0 - literal (uncompressed) cluster
+ *    1 - compressed cluster (for the head logical cluster)
+ *    2 - compressed cluster (for the other logical clusters)
+ *
+ * In detail,
+ *    0 - literal (uncompressed) cluster,
+ *        di_advise = 0
+ *        di_clusterofs = the literal data offset of the cluster
+ *        di_blkaddr = the blkaddr of the literal cluster
+ *
+ *    1 - compressed cluster (for the head logical cluster)
+ *        di_advise = 1
+ *        di_clusterofs = the decompressed data offset of the cluster
+ *        di_blkaddr = the blkaddr of the compressed cluster
+ *
+ *    2 - compressed cluster (for the other logical clusters)
+ *        di_advise = 2
+ *        di_clusterofs =
+ *           the decompressed data offset in its own head cluster
+ *        di_u.delta[0] = distance to its corresponding head cluster
+ *        di_u.delta[1] = distance to its corresponding tail cluster
+ *                (di_advise could be 0, 1 or 2)
+ */
+#define Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS        2
+#define Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT         0
+
+struct z_erofs_vle_decompressed_index {
+	__le16 di_advise;
+	/* where to decompress in the head cluster */
+	__le16 di_clusterofs;
+
+	union {
+		/* for the head cluster */
+		__le32 blkaddr;
+		/*
+		 * for the rest clusters
+		 * eg. for 4k page-sized cluster, maximum 4K*64k = 256M)
+		 * [0] - pointing to the head cluster
+		 * [1] - pointing to the tail cluster
+		 */
+		__le16 delta[2];
+	} di_u __packed;		/* 8 bytes */
+} __packed;
+
+#define Z_EROFS_VLE_EXTENT_ALIGN(size) round_up(size, \
+	sizeof(struct z_erofs_vle_decompressed_index))
+
+#define ondisk_extent_size(data_mapping_mode, count) \
+	((data_mapping_mode) == EROFS_INODE_LAYOUT_COMPRESSION ? \
+		(sizeof(struct erofs_extent_header) + \
+		sizeof(__u32) * le32_to_cpu(count)) : 0)
+
+/* dirent sorts in alphabet order, thus we can do binary search */
+struct erofs_dirent {
+	__le64 nid;     /*  0, node number */
+	__le16 nameoff; /*  8, start offset of file name */
+	__u8 file_type; /* 10, file type */
+	__u8 reserved;  /* 11, reserved */
+} __packed;
+
+/* file types used in inode_info->flags */
+enum {
+	EROFS_FT_UNKNOWN,
+	EROFS_FT_REG_FILE,
+	EROFS_FT_DIR,
+	EROFS_FT_CHRDEV,
+	EROFS_FT_BLKDEV,
+	EROFS_FT_FIFO,
+	EROFS_FT_SOCK,
+	EROFS_FT_SYMLINK,
+	EROFS_FT_MAX
+};
+
+#define EROFS_NAME_LEN      255
+
+/* check the EROFS on-disk layout strictly at compile time */
+static inline void erofs_check_ondisk_layout_definitions(void)
+{
+	BUILD_BUG_ON(sizeof(struct erofs_super_block) != 128);
+	BUILD_BUG_ON(sizeof(struct erofs_inode_v1) != 32);
+	BUILD_BUG_ON(sizeof(struct erofs_inode_v2) != 64);
+	BUILD_BUG_ON(sizeof(struct erofs_xattr_ibody_header) != 12);
+	BUILD_BUG_ON(sizeof(struct erofs_xattr_entry) != 4);
+	BUILD_BUG_ON(sizeof(struct erofs_extent_header) != 16);
+	BUILD_BUG_ON(sizeof(struct z_erofs_vle_decompressed_index) != 8);
+	BUILD_BUG_ON(sizeof(struct erofs_dirent) != 12);
+}
+
+#endif
+
-- 
1.9.1

^ permalink raw reply related

* [RFC PATCH 00/25] staging: erofs: introduce erofs file system
From: Gao Xiang @ 2018-07-24  2:36 UTC (permalink / raw)



This patchset includes the full EROFS file system.

It is the last patchset using LINUX_VERSION macro, and
the first patchset moving full implementation to
'drivers/staging/erofs' for the preliminary mainline upstreaming.


These commits have been merged as a part of larger patches:
 - erofs: fix unrecognized mount option message level
 - erofs: fix the potential integer overflow
 - erofs: fix match_table with a NULL terminated pattern
 - erofs: fix erofs_module_init & exit
 - erofs: introduce parse_options()
 - erofs: remove unused EROFS_XATTR_INDEX_ADVISE
 - erofs: fix to return correct value of alloc_inode
 - erofs: fix to do endian conversion correctly
 - erofs: fix missing endian conversion
 - erofs: fix to avoid potential overflow
 - erofs: fix compile error
 - erofs: add the missing header <linux/prefetch.h>
 - erofs: fix ifnullfree.cocci warnings


Chao Yu (3):
  staging: erofs: support special inode
  staging: erofs: introduce error injection infrastructure
  staging: erofs: support tracepoint

Gao Xiang (22):
  staging: erofs: add on-disk layout
  staging: erofs: add erofs in-memory stuffs
  staging: erofs: add super block operations
  staging: erofs: add raw address_space operations
  staging: erofs: add inode operations
  staging: erofs: add directory operations
  staging: erofs: add namei functions
  staging: erofs: definitions for kernel compatibility
  staging: erofs: update Kconfig and Makefile
  staging: erofs: introduce xattr & acl support
  staging: erofs: <linux/tagptr.h>: introduce tagged pointer
  staging: erofs: introduce pagevec for unzip subsystem
  staging: erofs: add erofs_map_blocks_iter
  staging: erofs: add erofs_allocpage
  staging: erofs: globalize prepare_bio and __submit_bio
  staging: erofs: introduce a customized LZ4 decompression
  staging: erofs: add a generic z_erofs VLE decompressor
  staging: erofs: introduce superblock registration
  staging: erofs: introduce erofs shrinker
  staging: erofs: introduce workstation for decompression
  staging: erofs: introduce VLE decompression support
  staging: erofs: introduce cached decompression

 drivers/staging/Kconfig                            |    2 +
 drivers/staging/Makefile                           |    1 +
 drivers/staging/erofs/Kconfig                      |  141 ++
 drivers/staging/erofs/Makefile                     |   13 +
 drivers/staging/erofs/data.c                       |  393 +++++
 drivers/staging/erofs/dir.c                        |  145 ++
 drivers/staging/erofs/erofs_fs.h                   |  271 ++++
 drivers/staging/erofs/include/linux/tagptr.h       |  110 ++
 drivers/staging/erofs/include/trace/events/erofs.h |  240 +++
 drivers/staging/erofs/inode.c                      |  334 ++++
 drivers/staging/erofs/internal.h                   |  578 +++++++
 drivers/staging/erofs/lz4defs.h                    |  227 +++
 drivers/staging/erofs/namei.c                      |  254 +++
 drivers/staging/erofs/staging.h                    |  133 ++
 drivers/staging/erofs/super.c                      |  660 ++++++++
 drivers/staging/erofs/unzip_lz4.c                  |  251 +++
 drivers/staging/erofs/unzip_pagevec.h              |  172 ++
 drivers/staging/erofs/unzip_vle.c                  | 1637 ++++++++++++++++++++
 drivers/staging/erofs/unzip_vle.h                  |  239 +++
 drivers/staging/erofs/unzip_vle_lz4.c              |  209 +++
 drivers/staging/erofs/utils.c                      |  270 ++++
 drivers/staging/erofs/xattr.c                      |  678 ++++++++
 drivers/staging/erofs/xattr.h                      |   93 ++
 23 files changed, 7051 insertions(+)
 create mode 100644 drivers/staging/erofs/Kconfig
 create mode 100644 drivers/staging/erofs/Makefile
 create mode 100644 drivers/staging/erofs/data.c
 create mode 100644 drivers/staging/erofs/dir.c
 create mode 100644 drivers/staging/erofs/erofs_fs.h
 create mode 100644 drivers/staging/erofs/include/linux/tagptr.h
 create mode 100644 drivers/staging/erofs/include/trace/events/erofs.h
 create mode 100644 drivers/staging/erofs/inode.c
 create mode 100644 drivers/staging/erofs/internal.h
 create mode 100644 drivers/staging/erofs/lz4defs.h
 create mode 100644 drivers/staging/erofs/namei.c
 create mode 100644 drivers/staging/erofs/staging.h
 create mode 100644 drivers/staging/erofs/super.c
 create mode 100644 drivers/staging/erofs/unzip_lz4.c
 create mode 100644 drivers/staging/erofs/unzip_pagevec.h
 create mode 100644 drivers/staging/erofs/unzip_vle.c
 create mode 100644 drivers/staging/erofs/unzip_vle.h
 create mode 100644 drivers/staging/erofs/unzip_vle_lz4.c
 create mode 100644 drivers/staging/erofs/utils.c
 create mode 100644 drivers/staging/erofs/xattr.c
 create mode 100644 drivers/staging/erofs/xattr.h

-- 
1.9.1

^ permalink raw reply

* [PATCH 5/5] Input: pxrc - flatten probe code
From: Dmitry Torokhov @ 2018-07-24  2:35 UTC (permalink / raw)
  To: Marcus Folkesson; +Cc: linux-input, linux-kernel, Alexey Khoroshilov
In-Reply-To: <20180724023520.2189-1-dmitry.torokhov@gmail.com>

Instead of splitting probe code into separate USB and input setup, flatten it.
This allows for easier inspection of order of set up steps, since the probe code
is reasonably small.

Move input-related initialization (phys) from USB block to input block.

Signed-off-by: Dmitry Torokhov <dmitry.torokhov@gmail.com>
---
 drivers/input/joystick/pxrc.c | 84 ++++++++++++++++++-------------------------
 1 file changed, 35 insertions(+), 49 deletions(-)

diff --git a/drivers/input/joystick/pxrc.c b/drivers/input/joystick/pxrc.c
index 887a0df9d9a7..ea2bf5951d67 100644
--- a/drivers/input/joystick/pxrc.c
+++ b/drivers/input/joystick/pxrc.c
@@ -3,7 +3,6 @@
  * Driver for Phoenix RC Flight Controller Adapter
  *
  * Copyright (C) 2018 Marcus Folkesson <marcus.folkesson@gmail.com>
- *
  */
 
 #include <linux/kernel.h>
@@ -16,8 +15,8 @@
 #include <linux/mutex.h>
 #include <linux/input.h>
 
-#define PXRC_VENDOR_ID	(0x1781)
-#define PXRC_PRODUCT_ID	(0x0898)
+#define PXRC_VENDOR_ID		0x1781
+#define PXRC_PRODUCT_ID		0x0898
 
 struct pxrc {
 	struct input_dev	*input;
@@ -118,59 +117,66 @@ static void pxrc_free_urb(void *_pxrc)
 	usb_free_urb(pxrc->urb);
 }
 
-static int pxrc_usb_init(struct pxrc *pxrc)
+static int pxrc_probe(struct usb_interface *intf,
+		      const struct usb_device_id *id)
 {
-	struct usb_device *udev = interface_to_usbdev(pxrc->intf);
+	struct usb_device *udev = interface_to_usbdev(intf);
+	struct pxrc *pxrc;
 	struct usb_endpoint_descriptor *epirq;
 	size_t xfer_size;
 	void *xfer_buf;
-	unsigned int pipe;
 	int error;
 
-	/* Set up the endpoint information */
-	/* This device only has an interrupt endpoint */
-	error = usb_find_common_endpoints(pxrc->intf->cur_altsetting,
+	/*
+	 * Locate the endpoint information. This device only has an
+	 * interrupt endpoint.
+	 */
+	error = usb_find_common_endpoints(intf->cur_altsetting,
 					  NULL, NULL, &epirq, NULL);
 	if (error) {
-		dev_err(&pxrc->intf->dev, "Could not find endpoint\n");
+		dev_err(&intf->dev, "Could not find endpoint\n");
 		return error;
 	}
 
-	xfer_size = usb_endpoint_maxp(epirq);
-	xfer_buf = devm_kmalloc(&pxrc->intf->dev, xfer_size, GFP_KERNEL);
-	if (!xfer_buf)
+	pxrc = devm_kzalloc(&intf->dev, sizeof(*pxrc), GFP_KERNEL);
+	if (!pxrc)
 		return -ENOMEM;
 
+	mutex_init(&pxrc->pm_mutex);
+	pxrc->intf = intf;
+
 	usb_set_intfdata(pxrc->intf, pxrc);
-	usb_make_path(udev, pxrc->phys, sizeof(pxrc->phys));
-	strlcat(pxrc->phys, "/input0", sizeof(pxrc->phys));
+
+	xfer_size = usb_endpoint_maxp(epirq);
+	xfer_buf = devm_kmalloc(&intf->dev, xfer_size, GFP_KERNEL);
+	if (!xfer_buf)
+		return -ENOMEM;
 
 	pxrc->urb = usb_alloc_urb(0, GFP_KERNEL);
 	if (!pxrc->urb)
 		return -ENOMEM;
 
-	error = devm_add_action_or_reset(&pxrc->intf->dev, pxrc_free_urb, pxrc);
+	error = devm_add_action_or_reset(&intf->dev, pxrc_free_urb, pxrc);
 	if (error)
 		return error;
 
-	pipe = usb_rcvintpipe(udev, epirq->bEndpointAddress),
-	usb_fill_int_urb(pxrc->urb, udev, pipe, xfer_buf, xfer_size,
-			 pxrc_usb_irq, pxrc, 1);
+	usb_fill_int_urb(pxrc->urb, udev,
+			 usb_rcvintpipe(udev, epirq->bEndpointAddress),
+			 xfer_buf, xfer_size, pxrc_usb_irq, pxrc, 1);
 
-	return 0;
-}
-
-static int pxrc_input_init(struct pxrc *pxrc)
-{
-	pxrc->input = devm_input_allocate_device(&pxrc->intf->dev);
-	if (pxrc->input == NULL) {
-		dev_err(&pxrc->intf->dev, "couldn't allocate input device\n");
+	pxrc->input = devm_input_allocate_device(&intf->dev);
+	if (!pxrc->input) {
+		dev_err(&intf->dev, "couldn't allocate input device\n");
 		return -ENOMEM;
 	}
 
 	pxrc->input->name = "PXRC Flight Controller Adapter";
+
+	usb_make_path(udev, pxrc->phys, sizeof(pxrc->phys));
+	strlcat(pxrc->phys, "/input0", sizeof(pxrc->phys));
 	pxrc->input->phys = pxrc->phys;
-	usb_to_input_id(interface_to_usbdev(pxrc->intf), &pxrc->input->id);
+
+	usb_to_input_id(udev, &pxrc->input->id);
 
 	pxrc->input->open = pxrc_open;
 	pxrc->input->close = pxrc_close;
@@ -186,27 +192,7 @@ static int pxrc_input_init(struct pxrc *pxrc)
 
 	input_set_drvdata(pxrc->input, pxrc);
 
-	return input_register_device(pxrc->input);
-}
-
-static int pxrc_probe(struct usb_interface *intf,
-		      const struct usb_device_id *id)
-{
-	struct pxrc *pxrc;
-	int error;
-
-	pxrc = devm_kzalloc(&intf->dev, sizeof(*pxrc), GFP_KERNEL);
-	if (!pxrc)
-		return -ENOMEM;
-
-	mutex_init(&pxrc->pm_mutex);
-	pxrc->intf = intf;
-
-	error = pxrc_usb_init(pxrc);
-	if (error)
-		return error;
-
-	error = pxrc_input_init(pxrc);
+	error = input_register_device(pxrc->input);
 	if (error)
 		return error;
 
-- 
2.11.0

^ permalink raw reply related

* [PATCH 4/5] Input: pxrc - do not store unneeded data in driver structure
From: Dmitry Torokhov @ 2018-07-24  2:35 UTC (permalink / raw)
  To: Marcus Folkesson; +Cc: linux-input, linux-kernel, Alexey Khoroshilov
In-Reply-To: <20180724023520.2189-1-dmitry.torokhov@gmail.com>

There is no need to store data buffer size, pointer to the buffer, or endpoint
address in pxrc structure, as they are either only needed during setup, or are
available from elsewhere.

Signed-off-by: Dmitry Torokhov <dmitry.torokhov@gmail.com>
---
 drivers/input/joystick/pxrc.c | 35 +++++++++++++++++------------------
 1 file changed, 17 insertions(+), 18 deletions(-)

diff --git a/drivers/input/joystick/pxrc.c b/drivers/input/joystick/pxrc.c
index 327b5ef8515f..887a0df9d9a7 100644
--- a/drivers/input/joystick/pxrc.c
+++ b/drivers/input/joystick/pxrc.c
@@ -25,15 +25,13 @@ struct pxrc {
 	struct urb		*urb;
 	struct mutex		pm_mutex;
 	bool			is_open;
-	__u8			epaddr;
 	char			phys[64];
-	unsigned char           *data;
-	size_t			bsize;
 };
 
 static void pxrc_usb_irq(struct urb *urb)
 {
 	struct pxrc *pxrc = urb->context;
+	u8 *data = urb->transfer_buffer;
 	int error;
 
 	switch (urb->status) {
@@ -61,15 +59,15 @@ static void pxrc_usb_irq(struct urb *urb)
 	}
 
 	if (urb->actual_length == 8) {
-		input_report_abs(pxrc->input, ABS_X, pxrc->data[0]);
-		input_report_abs(pxrc->input, ABS_Y, pxrc->data[2]);
-		input_report_abs(pxrc->input, ABS_RX, pxrc->data[3]);
-		input_report_abs(pxrc->input, ABS_RY, pxrc->data[4]);
-		input_report_abs(pxrc->input, ABS_RUDDER, pxrc->data[5]);
-		input_report_abs(pxrc->input, ABS_THROTTLE, pxrc->data[6]);
-		input_report_abs(pxrc->input, ABS_MISC, pxrc->data[7]);
-
-		input_report_key(pxrc->input, BTN_A, pxrc->data[1]);
+		input_report_abs(pxrc->input, ABS_X, data[0]);
+		input_report_abs(pxrc->input, ABS_Y, data[2]);
+		input_report_abs(pxrc->input, ABS_RX, data[3]);
+		input_report_abs(pxrc->input, ABS_RY, data[4]);
+		input_report_abs(pxrc->input, ABS_RUDDER, data[5]);
+		input_report_abs(pxrc->input, ABS_THROTTLE, data[6]);
+		input_report_abs(pxrc->input, ABS_MISC, data[7]);
+
+		input_report_key(pxrc->input, BTN_A, data[1]);
 	}
 
 exit:
@@ -124,6 +122,8 @@ static int pxrc_usb_init(struct pxrc *pxrc)
 {
 	struct usb_device *udev = interface_to_usbdev(pxrc->intf);
 	struct usb_endpoint_descriptor *epirq;
+	size_t xfer_size;
+	void *xfer_buf;
 	unsigned int pipe;
 	int error;
 
@@ -136,10 +136,9 @@ static int pxrc_usb_init(struct pxrc *pxrc)
 		return error;
 	}
 
-	pxrc->bsize = usb_endpoint_maxp(epirq);
-	pxrc->epaddr = epirq->bEndpointAddress;
-	pxrc->data = devm_kmalloc(&pxrc->intf->dev, pxrc->bsize, GFP_KERNEL);
-	if (!pxrc->data)
+	xfer_size = usb_endpoint_maxp(epirq);
+	xfer_buf = devm_kmalloc(&pxrc->intf->dev, xfer_size, GFP_KERNEL);
+	if (!xfer_buf)
 		return -ENOMEM;
 
 	usb_set_intfdata(pxrc->intf, pxrc);
@@ -154,8 +153,8 @@ static int pxrc_usb_init(struct pxrc *pxrc)
 	if (error)
 		return error;
 
-	pipe = usb_rcvintpipe(udev, pxrc->epaddr),
-	usb_fill_int_urb(pxrc->urb, udev, pipe, pxrc->data, pxrc->bsize,
+	pipe = usb_rcvintpipe(udev, epirq->bEndpointAddress),
+	usb_fill_int_urb(pxrc->urb, udev, pipe, xfer_buf, xfer_size,
 			 pxrc_usb_irq, pxrc, 1);
 
 	return 0;
-- 
2.11.0

^ permalink raw reply related

* [PATCH 3/5] Input: pxrc - move module device table closer to where it is used
From: Dmitry Torokhov @ 2018-07-24  2:35 UTC (permalink / raw)
  To: Marcus Folkesson; +Cc: linux-input, linux-kernel, Alexey Khoroshilov
In-Reply-To: <20180724023520.2189-1-dmitry.torokhov@gmail.com>

There is no need to have the device table first in the file.

Signed-off-by: Dmitry Torokhov <dmitry.torokhov@gmail.com>
---
 drivers/input/joystick/pxrc.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/drivers/input/joystick/pxrc.c b/drivers/input/joystick/pxrc.c
index 1560f0e39c34..327b5ef8515f 100644
--- a/drivers/input/joystick/pxrc.c
+++ b/drivers/input/joystick/pxrc.c
@@ -19,12 +19,6 @@
 #define PXRC_VENDOR_ID	(0x1781)
 #define PXRC_PRODUCT_ID	(0x0898)
 
-static const struct usb_device_id pxrc_table[] = {
-	{ USB_DEVICE(PXRC_VENDOR_ID, PXRC_PRODUCT_ID) },
-	{ }
-};
-MODULE_DEVICE_TABLE(usb, pxrc_table);
-
 struct pxrc {
 	struct input_dev	*input;
 	struct usb_interface	*intf;
@@ -277,6 +271,12 @@ static int pxrc_reset_resume(struct usb_interface *intf)
 	return pxrc_resume(intf);
 }
 
+static const struct usb_device_id pxrc_table[] = {
+	{ USB_DEVICE(PXRC_VENDOR_ID, PXRC_PRODUCT_ID) },
+	{ }
+};
+MODULE_DEVICE_TABLE(usb, pxrc_table);
+
 static struct usb_driver pxrc_driver = {
 	.name =		"pxrc",
 	.probe =	pxrc_probe,
-- 
2.11.0

^ permalink raw reply related

* [PATCH 2/5] Input: pxrc - fix freeing URB on device teardown
From: Dmitry Torokhov @ 2018-07-24  2:35 UTC (permalink / raw)
  To: Marcus Folkesson; +Cc: linux-input, linux-kernel, Alexey Khoroshilov
In-Reply-To: <20180724023520.2189-1-dmitry.torokhov@gmail.com>

URB is the only resource that is not managed, and thus is destroyed too early,
before we unregister input device and stop URB in pxrc_close(). To fix it let's
install custom devm handler to free the URB at the right time in devm unwind
sequence.

Signed-off-by: Dmitry Torokhov <dmitry.torokhov@gmail.com>
---
 drivers/input/joystick/pxrc.c | 66 ++++++++++++++++++++-----------------------
 1 file changed, 30 insertions(+), 36 deletions(-)

diff --git a/drivers/input/joystick/pxrc.c b/drivers/input/joystick/pxrc.c
index 000b9b7da744..1560f0e39c34 100644
--- a/drivers/input/joystick/pxrc.c
+++ b/drivers/input/joystick/pxrc.c
@@ -119,49 +119,52 @@ static void pxrc_close(struct input_dev *input)
 	mutex_unlock(&pxrc->pm_mutex);
 }
 
+static void pxrc_free_urb(void *_pxrc)
+{
+	struct pxrc *pxrc = _pxrc;
+
+	usb_free_urb(pxrc->urb);
+}
+
 static int pxrc_usb_init(struct pxrc *pxrc)
 {
 	struct usb_device *udev = interface_to_usbdev(pxrc->intf);
 	struct usb_endpoint_descriptor *epirq;
 	unsigned int pipe;
-	int retval;
+	int error;
 
 	/* Set up the endpoint information */
 	/* This device only has an interrupt endpoint */
-	retval = usb_find_common_endpoints(pxrc->intf->cur_altsetting,
-			NULL, NULL, &epirq, NULL);
-	if (retval) {
-		dev_err(&pxrc->intf->dev,
-			"Could not find endpoint\n");
-		goto error;
+	error = usb_find_common_endpoints(pxrc->intf->cur_altsetting,
+					  NULL, NULL, &epirq, NULL);
+	if (error) {
+		dev_err(&pxrc->intf->dev, "Could not find endpoint\n");
+		return error;
 	}
 
 	pxrc->bsize = usb_endpoint_maxp(epirq);
 	pxrc->epaddr = epirq->bEndpointAddress;
 	pxrc->data = devm_kmalloc(&pxrc->intf->dev, pxrc->bsize, GFP_KERNEL);
-	if (!pxrc->data) {
-		retval = -ENOMEM;
-		goto error;
-	}
+	if (!pxrc->data)
+		return -ENOMEM;
 
 	usb_set_intfdata(pxrc->intf, pxrc);
 	usb_make_path(udev, pxrc->phys, sizeof(pxrc->phys));
 	strlcat(pxrc->phys, "/input0", sizeof(pxrc->phys));
 
 	pxrc->urb = usb_alloc_urb(0, GFP_KERNEL);
-	if (!pxrc->urb) {
-		retval = -ENOMEM;
-		goto error;
-	}
+	if (!pxrc->urb)
+		return -ENOMEM;
+
+	error = devm_add_action_or_reset(&pxrc->intf->dev, pxrc_free_urb, pxrc);
+	if (error)
+		return error;
 
 	pipe = usb_rcvintpipe(udev, pxrc->epaddr),
 	usb_fill_int_urb(pxrc->urb, udev, pipe, pxrc->data, pxrc->bsize,
 			 pxrc_usb_irq, pxrc, 1);
 
-error:
-	return retval;
-
-
+	return 0;
 }
 
 static int pxrc_input_init(struct pxrc *pxrc)
@@ -197,7 +200,7 @@ static int pxrc_probe(struct usb_interface *intf,
 		      const struct usb_device_id *id)
 {
 	struct pxrc *pxrc;
-	int retval;
+	int error;
 
 	pxrc = devm_kzalloc(&intf->dev, sizeof(*pxrc), GFP_KERNEL);
 	if (!pxrc)
@@ -206,29 +209,20 @@ static int pxrc_probe(struct usb_interface *intf,
 	mutex_init(&pxrc->pm_mutex);
 	pxrc->intf = intf;
 
-	retval = pxrc_usb_init(pxrc);
-	if (retval)
-		goto error;
+	error = pxrc_usb_init(pxrc);
+	if (error)
+		return error;
 
-	retval = pxrc_input_init(pxrc);
-	if (retval)
-		goto err_free_urb;
+	error = pxrc_input_init(pxrc);
+	if (error)
+		return error;
 
 	return 0;
-
-err_free_urb:
-	usb_free_urb(pxrc->urb);
-
-error:
-	return retval;
 }
 
 static void pxrc_disconnect(struct usb_interface *intf)
 {
-	struct pxrc *pxrc = usb_get_intfdata(intf);
-
-	usb_free_urb(pxrc->urb);
-	usb_set_intfdata(intf, NULL);
+	/* All driver resources are devm-managed. */
 }
 
 static int pxrc_suspend(struct usb_interface *intf, pm_message_t message)
-- 
2.11.0

^ permalink raw reply related

* [PATCH 1/5] Input: pxrc - do not store USB device in private struct
From: Dmitry Torokhov @ 2018-07-24  2:35 UTC (permalink / raw)
  To: Marcus Folkesson; +Cc: linux-input, linux-kernel, Alexey Khoroshilov

From: Marcus Folkesson <marcus.folkesson@gmail.com>

The USB device is only needed during setup, so put it back after
initialization and do not store it in our private struct.

Also, the USB device is a parent of USB interface so our driver
model rules ensure that USB device should not disappear while
interface device is still there; there is no need to take reference
to the USB device.

Reported-by: Alexey Khoroshilov <khoroshilov@ispras.ru>
Signed-off-by: Marcus Folkesson <marcus.folkesson@gmail.com>
Patchwork-Id: 10526903
Signed-off-by: Dmitry Torokhov <dmitry.torokhov@gmail.com>
---
 drivers/input/joystick/pxrc.c | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/drivers/input/joystick/pxrc.c b/drivers/input/joystick/pxrc.c
index 07a0dbd3ced2..000b9b7da744 100644
--- a/drivers/input/joystick/pxrc.c
+++ b/drivers/input/joystick/pxrc.c
@@ -27,7 +27,6 @@ MODULE_DEVICE_TABLE(usb, pxrc_table);
 
 struct pxrc {
 	struct input_dev	*input;
-	struct usb_device	*udev;
 	struct usb_interface	*intf;
 	struct urb		*urb;
 	struct mutex		pm_mutex;
@@ -122,6 +121,7 @@ static void pxrc_close(struct input_dev *input)
 
 static int pxrc_usb_init(struct pxrc *pxrc)
 {
+	struct usb_device *udev = interface_to_usbdev(pxrc->intf);
 	struct usb_endpoint_descriptor *epirq;
 	unsigned int pipe;
 	int retval;
@@ -145,7 +145,7 @@ static int pxrc_usb_init(struct pxrc *pxrc)
 	}
 
 	usb_set_intfdata(pxrc->intf, pxrc);
-	usb_make_path(pxrc->udev, pxrc->phys, sizeof(pxrc->phys));
+	usb_make_path(udev, pxrc->phys, sizeof(pxrc->phys));
 	strlcat(pxrc->phys, "/input0", sizeof(pxrc->phys));
 
 	pxrc->urb = usb_alloc_urb(0, GFP_KERNEL);
@@ -154,9 +154,9 @@ static int pxrc_usb_init(struct pxrc *pxrc)
 		goto error;
 	}
 
-	pipe = usb_rcvintpipe(pxrc->udev, pxrc->epaddr),
-	usb_fill_int_urb(pxrc->urb, pxrc->udev, pipe, pxrc->data, pxrc->bsize,
-						pxrc_usb_irq, pxrc, 1);
+	pipe = usb_rcvintpipe(udev, pxrc->epaddr),
+	usb_fill_int_urb(pxrc->urb, udev, pipe, pxrc->data, pxrc->bsize,
+			 pxrc_usb_irq, pxrc, 1);
 
 error:
 	return retval;
@@ -174,7 +174,7 @@ static int pxrc_input_init(struct pxrc *pxrc)
 
 	pxrc->input->name = "PXRC Flight Controller Adapter";
 	pxrc->input->phys = pxrc->phys;
-	usb_to_input_id(pxrc->udev, &pxrc->input->id);
+	usb_to_input_id(interface_to_usbdev(pxrc->intf), &pxrc->input->id);
 
 	pxrc->input->open = pxrc_open;
 	pxrc->input->close = pxrc_close;
@@ -204,7 +204,6 @@ static int pxrc_probe(struct usb_interface *intf,
 		return -ENOMEM;
 
 	mutex_init(&pxrc->pm_mutex);
-	pxrc->udev = usb_get_dev(interface_to_usbdev(intf));
 	pxrc->intf = intf;
 
 	retval = pxrc_usb_init(pxrc);
-- 
2.11.0

^ permalink raw reply related

* [meta-filesystems][PATCH] yaffs2-utils: update to latest master
From: Yi Zhao @ 2018-07-24  2:34 UTC (permalink / raw)
  To: openembedded-devel

License-Update: Modify the line number for mkyaffs2image.c license.
                Add license for mkyaffsimage.c since it also be built.

Signed-off-by: Yi Zhao <yi.zhao@windriver.com>
---
 meta-filesystems/recipes-filesystems/yaffs2/yaffs2-utils_git.bb | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/meta-filesystems/recipes-filesystems/yaffs2/yaffs2-utils_git.bb b/meta-filesystems/recipes-filesystems/yaffs2/yaffs2-utils_git.bb
index 317965e..9ca6d9b 100644
--- a/meta-filesystems/recipes-filesystems/yaffs2/yaffs2-utils_git.bb
+++ b/meta-filesystems/recipes-filesystems/yaffs2/yaffs2-utils_git.bb
@@ -5,7 +5,9 @@ SECTION = "base"
 HOMEPAGE = "http://www.yaffs.net"
 
 LICENSE = "GPLv2"
-LIC_FILES_CHKSUM = "file://utils/mkyaffs2image.c;beginline=12;endline=14;md5=5f5464f9b3e981ca574e65b00e438561"
+LIC_FILES_CHKSUM = "file://utils/mkyaffs2image.c;beginline=11;endline=13;md5=5f5464f9b3e981ca574e65b00e438561 \
+                    file://utils/mkyaffsimage.c;beginline=10;endline=12;md5=5f5464f9b3e981ca574e65b00e438561 \
+                    "
 
 PV = "0.0+git${SRCPV}"
 
@@ -17,7 +19,7 @@ SRC_URI = "git://www.aleph1.co.uk/yaffs2;protocol=git;branch=master \
            file://0001-define-loff_t-if-not-already-defined.patch \
            "
 
-SRCREV = "bc76682d93955cfb33051beb503ad9f8a5450578"
+SRCREV = "0065378b27638ee07352282b51b596fabcac26e4"
 
 UPSTREAM_CHECK_COMMITS = "1"
 
-- 
2.7.4



^ permalink raw reply related

* [meta-networking][PATCH] snort: upgrade 2.9.7.5 -> 2.9.11.1
From: Yi Zhao @ 2018-07-24  2:33 UTC (permalink / raw)
  To: openembedded-devel

* Update SRC_URI

* Drop 0001-fix-do_package-failed-since-snort-2.9.7.0.patch since the
  issue had been fixed upstream.

* Drop the following patches and merge their modifications to
  disable-run-test-program-while-cross-compiling.patch:
    disable-dap-address-space-id.patch
    disable-daq-flow-id.patch
    disable-daq-verdict-retry.patch
    disable-inaddr-none.patch

* Add disable-run-test-program-while-cross-compiling.patch to fix
  configure error: cannot run test program while cross compiling.

  The variables have_daq_real_addresses, have_daq_ext_modflow,
  have_daq_queryflow, have_daq_data_channel_flags and
  have_daq_data_channel_separate_ip_versions are set to no since they
  are for daq 2.2.2 but we have daq 2.0.6 in meta-networking layer.

Signed-off-by: Yi Zhao <yi.zhao@windriver.com>
---
 ...fix-do_package-failed-since-snort-2.9.7.0.patch |  33 ---
 .../snort/snort/disable-dap-address-space-id.patch |  62 -----
 .../snort/snort/disable-daq-flow-id.patch          |  60 -----
 .../snort/snort/disable-daq-verdict-retry.patch    |  60 -----
 .../snort/snort/disable-inaddr-none.patch          |  85 -------
 ...le-run-test-program-while-cross-compiling.patch | 254 +++++++++++++++++++++
 .../snort/{snort_2.9.7.5.bb => snort_2.9.11.1.bb}  |  12 +-
 7 files changed, 258 insertions(+), 308 deletions(-)
 delete mode 100644 meta-networking/recipes-connectivity/snort/snort/0001-fix-do_package-failed-since-snort-2.9.7.0.patch
 delete mode 100644 meta-networking/recipes-connectivity/snort/snort/disable-dap-address-space-id.patch
 delete mode 100644 meta-networking/recipes-connectivity/snort/snort/disable-daq-flow-id.patch
 delete mode 100644 meta-networking/recipes-connectivity/snort/snort/disable-daq-verdict-retry.patch
 delete mode 100644 meta-networking/recipes-connectivity/snort/snort/disable-inaddr-none.patch
 create mode 100644 meta-networking/recipes-connectivity/snort/snort/disable-run-test-program-while-cross-compiling.patch
 rename meta-networking/recipes-connectivity/snort/{snort_2.9.7.5.bb => snort_2.9.11.1.bb} (89%)

diff --git a/meta-networking/recipes-connectivity/snort/snort/0001-fix-do_package-failed-since-snort-2.9.7.0.patch b/meta-networking/recipes-connectivity/snort/snort/0001-fix-do_package-failed-since-snort-2.9.7.0.patch
deleted file mode 100644
index 047caec..0000000
--- a/meta-networking/recipes-connectivity/snort/snort/0001-fix-do_package-failed-since-snort-2.9.7.0.patch
+++ /dev/null
@@ -1,33 +0,0 @@
-From 4335c4c61877c0b65ff4ec767e8ad3ad8c73c32d Mon Sep 17 00:00:00 2001
-From: Zhiquan Li <zhiquan.li@windriver.com>
-Date: Fri, 13 Nov 2015 17:40:24 +0800
-Subject: [PATCH] fix do_package failed since snort 2.9.7.0
-
-Remove redundant '/' as a workround for below error:
-/lib/rpm/bin/debugedit: canonicalization unexpectedly shrank by one character
-
-It's a rpm debugedit bug:
-* http://sourceforge.net/p/snort/mailman/message/34130268/
-* https://bugzilla.redhat.com/show_bug.cgi?id=304121
-
-Upstream-Status:Inappropriate [embedded specific]
-
-Signed-off-by: Zhiquan Li <zhiquan.li@windriver.com>
-
----
- src/snort.h | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/src/snort.h b/src/snort.h
-index 8308e57..499d157 100644
---- a/src/snort.h
-+++ b/src/snort.h
-@@ -39,7 +39,7 @@
- #include "sfdaq.h"
- #include "sf_types.h"
- #include "sfutil/sflsq.h"
--#include "sfutil//sfActionQueue.h"
-+#include "sfutil/sfActionQueue.h"
- #include "profiler.h"
- #include "rules.h"
- #include "treenodes.h"
diff --git a/meta-networking/recipes-connectivity/snort/snort/disable-dap-address-space-id.patch b/meta-networking/recipes-connectivity/snort/snort/disable-dap-address-space-id.patch
deleted file mode 100644
index 6aa0b73..0000000
--- a/meta-networking/recipes-connectivity/snort/snort/disable-dap-address-space-id.patch
+++ /dev/null
@@ -1,62 +0,0 @@
-From 5a53260a819e2873f58165e96852529719101c53 Mon Sep 17 00:00:00 2001
-From: Chunrong Guo <B40290@freescale.com>
-Date: Mon, 4 Nov 2013 10:39:56 +0800
-Subject: [PATCH] snort: add recipe
-
-Upstream-Status:Inappropriate [embedded specific]
-
-fix the below error:
-checking for dap address space id... configure:
-configure: error: cannot run test program while cross compiling
-
-Signed-off-by: Chunrong Guo <B40290@freescale.com>
-
----
- configure.in | 34 +++++++++++++++++-----------------
- 1 file changed, 17 insertions(+), 17 deletions(-)
-
-diff --git a/configure.in b/configure.in
-index 6fe5d68..e7636ce 100644
---- a/configure.in
-+++ b/configure.in
-@@ -698,23 +698,23 @@ if test "x$ac_cv_func_daq_dp_add_dc" = "xyes"; then
- 
- fi
- 
--AC_MSG_CHECKING([for daq address space ID])
--AC_RUN_IFELSE(
--[AC_LANG_PROGRAM(
--[[
--#include <daq.h>
--]],
--[[
--   DAQ_PktHdr_t hdr;
--   hdr.address_space_id = 0;
--]])],
--[have_daq_address_space_id="yes"],
--[have_daq_address_space_id="no"])
--AC_MSG_RESULT($have_daq_address_space_id)
--if test "x$have_daq_address_space_id" = "xyes"; then
--    AC_DEFINE([HAVE_DAQ_ADDRESS_SPACE_ID],[1],
--        [DAQ version supports address space ID in header.])
--fi
-+#AC_MSG_CHECKING([for daq address space ID])
-+#AC_RUN_IFELSE(
-+#[AC_LANG_PROGRAM(
-+#[[
-+##include <daq.h>
-+#]],
-+#[[
-+#   DAQ_PktHdr_t hdr;
-+#   hdr.address_space_id = 0;
-+#]])],
-+have_daq_address_space_id="yes"
-+#[have_daq_address_space_id="no"])
-+#AC_MSG_RESULT($have_daq_address_space_id)
-+#if test "x$have_daq_address_space_id" = "xyes"; then
-+#    AC_DEFINE([HAVE_DAQ_ADDRESS_SPACE_ID],[1],
-+#        [DAQ version supports address space ID in header.])
-+#fi
- 
- AC_MSG_CHECKING([for daq flow ID])
- AC_RUN_IFELSE(
diff --git a/meta-networking/recipes-connectivity/snort/snort/disable-daq-flow-id.patch b/meta-networking/recipes-connectivity/snort/snort/disable-daq-flow-id.patch
deleted file mode 100644
index 04df37c..0000000
--- a/meta-networking/recipes-connectivity/snort/snort/disable-daq-flow-id.patch
+++ /dev/null
@@ -1,60 +0,0 @@
-From 5aa42a3137c657b5eddd4608343aa1854bd7cd27 Mon Sep 17 00:00:00 2001
-From: Zhiquan Li <zhiquan.li@windriver.com>
-Date: Mon, 16 Nov 2015 10:23:00 +0800
-Subject: [PATCH] snort: 2.9.6.0 -> 2.9.7.5
-
-Upstream-Status:Inappropriate [embedded specific]
-
-fix the below error:
-checking for daq flow ID... configure:
-configure: error: cannot run test program while cross compiling
-
----
- configure.in | 34 +++++++++++++++++-----------------
- 1 file changed, 17 insertions(+), 17 deletions(-)
-
-diff --git a/configure.in b/configure.in
-index e7636ce..2fbd298 100644
---- a/configure.in
-+++ b/configure.in
-@@ -716,23 +716,23 @@ have_daq_address_space_id="yes"
- #        [DAQ version supports address space ID in header.])
- #fi
- 
--AC_MSG_CHECKING([for daq flow ID])
--AC_RUN_IFELSE(
--[AC_LANG_PROGRAM(
--[[
--#include <daq.h>
--]],
--[[
--   DAQ_PktHdr_t hdr;
--   hdr.flow_id = 0;
--]])],
--[have_daq_flow_id="yes"],
--[have_daq_flow_id="no"])
--AC_MSG_RESULT($have_daq_flow_id)
--if test "x$have_daq_flow_id" = "xyes"; then
--    AC_DEFINE([HAVE_DAQ_FLOW_ID],[1],
--        [DAQ version supports flow ID in header.])
--fi
-+#AC_MSG_CHECKING([for daq flow ID])
-+#AC_RUN_IFELSE(
-+#[AC_LANG_PROGRAM(
-+#[[
-+##include <daq.h>
-+#]],
-+#[[
-+#   DAQ_PktHdr_t hdr;
-+#   hdr.flow_id = 0;
-+#]])],
-+have_daq_flow_id="yes"
-+#[have_daq_flow_id="no"])
-+#AC_MSG_RESULT($have_daq_flow_id)
-+#if test "x$have_daq_flow_id" = "xyes"; then
-+#    AC_DEFINE([HAVE_DAQ_FLOW_ID],[1],
-+#        [DAQ version supports flow ID in header.])
-+#fi
- 
- AC_MSG_CHECKING([for DAQ_VERDICT_RETRY])
- AC_RUN_IFELSE(
diff --git a/meta-networking/recipes-connectivity/snort/snort/disable-daq-verdict-retry.patch b/meta-networking/recipes-connectivity/snort/snort/disable-daq-verdict-retry.patch
deleted file mode 100644
index 15958a7..0000000
--- a/meta-networking/recipes-connectivity/snort/snort/disable-daq-verdict-retry.patch
+++ /dev/null
@@ -1,60 +0,0 @@
-From b916443d43f2e1eeacfbed0033274a0270ef634d Mon Sep 17 00:00:00 2001
-From: Zhiquan Li <zhiquan.li@windriver.com>
-Date: Mon, 16 Nov 2015 10:23:00 +0800
-Subject: [PATCH] snort: 2.9.6.0 -> 2.9.7.5
-
-Upstream-Status:Inappropriate [embedded specific]
-
-fix the below error:
-checking for DAQ_VERDICT_RETRY... configure:
-configure: error: cannot run test program while cross compiling
-
----
- configure.in | 34 +++++++++++++++++-----------------
- 1 file changed, 17 insertions(+), 17 deletions(-)
-
-diff --git a/configure.in b/configure.in
-index 2fbd298..62573a8 100644
---- a/configure.in
-+++ b/configure.in
-@@ -734,23 +734,23 @@ have_daq_flow_id="yes"
- #        [DAQ version supports flow ID in header.])
- #fi
- 
--AC_MSG_CHECKING([for DAQ_VERDICT_RETRY])
--AC_RUN_IFELSE(
--[AC_LANG_PROGRAM(
--[[
--#include <daq.h>
--]],
--[[
--   DAQ_Verdict verdict;
--   verdict = DAQ_VERDICT_RETRY;
--]])],
--[have_daq_verdict_retry="yes"],
--[have_daq_verdict_retry="no"])
--AC_MSG_RESULT($have_daq_verdict_retry)
--if test "x$have_daq_verdict_retry" = "xyes"; then
--    AC_DEFINE([HAVE_DAQ_VERDICT_RETRY],[1],
--        [DAQ version supports DAQ_VERDICT_RETRY in DAQ_Verdict.])
--fi
-+#AC_MSG_CHECKING([for DAQ_VERDICT_RETRY])
-+#AC_RUN_IFELSE(
-+#[AC_LANG_PROGRAM(
-+#[[
-+##include <daq.h>
-+#]],
-+#[[
-+#   DAQ_Verdict verdict;
-+#   verdict = DAQ_VERDICT_RETRY;
-+#]])],
-+have_daq_verdict_retry="yes"
-+#[have_daq_verdict_retry="no"])
-+#AC_MSG_RESULT($have_daq_verdict_retry)
-+#if test "x$have_daq_verdict_retry" = "xyes"; then
-+#    AC_DEFINE([HAVE_DAQ_VERDICT_RETRY],[1],
-+#        [DAQ version supports DAQ_VERDICT_RETRY in DAQ_Verdict.])
-+#fi
- 
- # any sparc platform has to have this one defined.
- AC_MSG_CHECKING(for sparc)
diff --git a/meta-networking/recipes-connectivity/snort/snort/disable-inaddr-none.patch b/meta-networking/recipes-connectivity/snort/snort/disable-inaddr-none.patch
deleted file mode 100644
index 79e9f07..0000000
--- a/meta-networking/recipes-connectivity/snort/snort/disable-inaddr-none.patch
+++ /dev/null
@@ -1,85 +0,0 @@
-From a1fdbced4166cb0f35d23b63a59312d86860485a Mon Sep 17 00:00:00 2001
-From: Chunrong Guo <B40290@freescale.com>
-Date: Mon, 4 Nov 2013 10:39:56 +0800
-Subject: [PATCH] snort: add recipe
-
-Upstream-Status: Inappropriate [embedded specific]
-
-fix the below error:
-checking for INADDR_NONE... configure:
-configure: error: cannot run test program while cross compiling
-
-Signed-off-by: Chunrong Guo <B40290@freescale.com>
-
----
- configure.in | 50 ++++++++++++++++----------------------------------
- 1 file changed, 16 insertions(+), 34 deletions(-)
-
-diff --git a/configure.in b/configure.in
-index 20d15eb..6fe5d68 100644
---- a/configure.in
-+++ b/configure.in
-@@ -274,25 +274,7 @@ AC_CHECK_TYPES([int8_t,int16_t,int32_t,int64_t])
- AC_CHECK_TYPES([boolean])
- 
- # In case INADDR_NONE is not defined (like on Solaris)
--have_inaddr_none="no"
--AC_MSG_CHECKING([for INADDR_NONE])
--AC_RUN_IFELSE(
--[AC_LANG_PROGRAM(
--[[
--#include <sys/types.h>
--#include <netinet/in.h>
--#include <arpa/inet.h>
--]],
--[[
--	if (inet_addr("10,5,2") == INADDR_NONE);
--    return 0;
--]])],
--[have_inaddr_none="yes"],
--[have_inaddr_none="no"])
--AC_MSG_RESULT($have_inaddr_none)
--if test "x$have_inaddr_none" = "xno"; then
--	AC_DEFINE([INADDR_NONE],[-1],[For INADDR_NONE definition])
--fi
-+have_inaddr_none="yes"
- 
- AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
- #include <stdio.h>
-@@ -421,21 +403,21 @@ if test "x$LPCAP" = "xno"; then
-   fi
- fi
- 
--AC_MSG_CHECKING([for pcap_lex_destroy])
--AC_RUN_IFELSE(
--[AC_LANG_PROGRAM(
--[[
--#include <pcap.h>
--]],
--[[
--   pcap_lex_destroy();
--]])],
--[have_pcap_lex_destroy="yes"],
--[have_pcap_lex_destroy="no"])
--AC_MSG_RESULT($have_pcap_lex_destroy)
--if test "x$have_pcap_lex_destroy" = "xyes"; then
--    AC_DEFINE([HAVE_PCAP_LEX_DESTROY],[1],[Can cleanup lex buffer stack created by pcap bpf filter])
--fi
-+#AC_MSG_CHECKING([for pcap_lex_destroy])
-+#AC_RUN_IFELSE(
-+#[AC_LANG_PROGRAM(
-+#[[
-+##include <pcap.h>
-+#]],
-+#[[
-+#   pcap_lex_destroy();
-+#]])],
-+have_pcap_lex_destroy="yes"
-+#[have_pcap_lex_destroy="no"])
-+#AC_MSG_RESULT($have_pcap_lex_destroy)
-+#if test "x$have_pcap_lex_destroy" = "xyes"; then
-+#    AC_DEFINE([HAVE_PCAP_LEX_DESTROY],[1],[Can cleanup lex buffer stack created by pcap bpf filter])
-+#fi
- 
- AC_MSG_CHECKING([for pcap_lib_version])
- AC_LINK_IFELSE(
diff --git a/meta-networking/recipes-connectivity/snort/snort/disable-run-test-program-while-cross-compiling.patch b/meta-networking/recipes-connectivity/snort/snort/disable-run-test-program-while-cross-compiling.patch
new file mode 100644
index 0000000..037962f
--- /dev/null
+++ b/meta-networking/recipes-connectivity/snort/snort/disable-run-test-program-while-cross-compiling.patch
@@ -0,0 +1,254 @@
+From dc390837bc7adc205bb955d0922040a7e365d8dd Mon Sep 17 00:00:00 2001
+From: Yi Zhao <yi.zhao@windriver.com>
+Date: Mon, 23 Jul 2018 13:54:07 +0800
+Subject: [PATCH] Disable run test program while cross compiling
+
+fix configure error:
+configure: error: cannot run test program while cross compiling
+
+Upstream-Status:Inappropriate [embedded specific]
+
+Signed-off-by: Yi Zhao <yi.zhao@windriver.com>
+---
+ configure.in | 148 +++++------------------------------------------------------
+ 1 file changed, 12 insertions(+), 136 deletions(-)
+
+diff --git a/configure.in b/configure.in
+index 4b3a5db..a6c5498 100644
+--- a/configure.in
++++ b/configure.in
+@@ -284,19 +284,7 @@ AC_CHECK_TYPES([boolean])
+ # In case INADDR_NONE is not defined (like on Solaris)
+ have_inaddr_none="no"
+ AC_MSG_CHECKING([for INADDR_NONE])
+-AC_RUN_IFELSE(
+-[AC_LANG_PROGRAM(
+-[[
+-#include <sys/types.h>
+-#include <netinet/in.h>
+-#include <arpa/inet.h>
+-]],
+-[[
+-	if (inet_addr("10,5,2") == INADDR_NONE);
+-    return 0;
+-]])],
+-[have_inaddr_none="yes"],
+-[have_inaddr_none="no"])
++have_inaddr_none="yes"
+ AC_MSG_RESULT($have_inaddr_none)
+ if test "x$have_inaddr_none" = "xno"; then
+ 	AC_DEFINE([INADDR_NONE],[-1],[For INADDR_NONE definition])
+@@ -429,16 +417,7 @@ if test "x$LPCAP" = "xno"; then
+ fi
+ 
+ AC_MSG_CHECKING([for pcap_lex_destroy])
+-AC_RUN_IFELSE(
+-[AC_LANG_PROGRAM(
+-[[
+-#include <pcap.h>
+-]],
+-[[
+-   pcap_lex_destroy();
+-]])],
+-[have_pcap_lex_destroy="yes"],
+-[have_pcap_lex_destroy="no"])
++have_pcap_lex_destroy="yes"
+ AC_MSG_RESULT($have_pcap_lex_destroy)
+ if test "x$have_pcap_lex_destroy" = "xyes"; then
+     AC_DEFINE([HAVE_PCAP_LEX_DESTROY],[1],[Can cleanup lex buffer stack created by pcap bpf filter])
+@@ -716,17 +695,7 @@ fi
+ AC_CHECK_FUNCS([daq_hup_apply] [daq_acquire_with_meta] [daq_dp_add_dc])
+ 
+ AC_MSG_CHECKING([for daq real addresses])
+-AC_RUN_IFELSE(
+-[AC_LANG_PROGRAM(
+-[[
+-#include <daq.h>
+-]],
+-[[
+-   DAQ_PktHdr_t hdr;
+-   hdr.n_real_dPort = 0;
+-]])],
+-[have_daq_real_addresses="yes"],
+-[have_daq_real_addresses="no"])
++have_daq_real_addresses="no"
+ AC_MSG_RESULT($have_daq_real_addresses)
+ if test "x$have_daq_real_addresses" = "xyes"; then
+     AC_DEFINE([HAVE_DAQ_REAL_ADDRESSES],[1],
+@@ -754,17 +723,7 @@ if test "x$ac_cv_func_daq_dp_add_dc" = "xyes"; then
+ fi
+ 
+ AC_MSG_CHECKING([for daq address space ID])
+-AC_RUN_IFELSE(
+-[AC_LANG_PROGRAM(
+-[[
+-#include <daq.h>
+-]],
+-[[
+-   DAQ_PktHdr_t hdr;
+-   hdr.address_space_id = 0;
+-]])],
+-[have_daq_address_space_id="yes"],
+-[have_daq_address_space_id="no"])
++have_daq_address_space_id="yes"
+ AC_MSG_RESULT($have_daq_address_space_id)
+ if test "x$have_daq_address_space_id" = "xyes"; then
+     AC_DEFINE([HAVE_DAQ_ADDRESS_SPACE_ID],[1],
+@@ -772,17 +731,7 @@ if test "x$have_daq_address_space_id" = "xyes"; then
+ fi
+ 
+ AC_MSG_CHECKING([for daq flow ID])
+-AC_RUN_IFELSE(
+-[AC_LANG_PROGRAM(
+-[[
+-#include <daq.h>
+-]],
+-[[
+-   DAQ_PktHdr_t hdr;
+-   hdr.flow_id = 0;
+-]])],
+-[have_daq_flow_id="yes"],
+-[have_daq_flow_id="no"])
++have_daq_flow_id="yes"
+ AC_MSG_RESULT($have_daq_flow_id)
+ if test "x$have_daq_flow_id" = "xyes"; then
+     AC_DEFINE([HAVE_DAQ_FLOW_ID],[1],
+@@ -790,19 +739,7 @@ if test "x$have_daq_flow_id" = "xyes"; then
+ fi
+ 
+ AC_MSG_CHECKING([for daq extended flow modifiers])
+-AC_RUN_IFELSE(
+-[AC_LANG_PROGRAM(
+-[[
+-#include <daq.h>
+-]],
+-[[
+-   DAQ_ModFlow_t mod;
+-   mod.type = 0;
+-   mod.length = 0;
+-   mod.value = NULL;
+-]])],
+-[have_daq_ext_modflow="yes"],
+-[have_daq_ext_modflow="no"])
++have_daq_ext_modflow="no"
+ AC_MSG_RESULT($have_daq_ext_modflow)
+ if test "x$have_daq_ext_modflow" = "xyes"; then
+     CCONFIGFLAGS="${CCONFIGFLAGS} -DHAVE_DAQ_EXT_MODFLOW"
+@@ -811,19 +748,7 @@ if test "x$have_daq_ext_modflow" = "xyes"; then
+ fi
+ 
+ AC_MSG_CHECKING([for daq query flow])
+-AC_RUN_IFELSE(
+-[AC_LANG_PROGRAM(
+-[[
+-#include <daq.h>
+-]],
+-[[
+-   DAQ_QueryFlow_t mod;
+-   mod.type = 0;
+-   mod.length = 0;
+-   mod.value = NULL;
+-]])],
+-[have_daq_queryflow="yes"],
+-[have_daq_queryflow="no"])
++have_daq_queryflow="no"
+ AC_MSG_RESULT($have_daq_queryflow)
+ if test "x$have_daq_queryflow" = "xyes"; then
+     CCONFIGFLAGS="${CCONFIGFLAGS} -DHAVE_DAQ_QUERYFLOW"
+@@ -832,16 +757,7 @@ if test "x$have_daq_queryflow" = "xyes"; then
+ fi
+ 
+ AC_MSG_CHECKING([for daq data channel flags])
+-AC_RUN_IFELSE(
+-[AC_LANG_PROGRAM(
+-[[
+-#include <daq.h>
+-]],
+-[[
+-   DAQ_Data_Channel_Params_t params;
+-]])],
+-[have_daq_data_channel_flags="yes"],
+-[have_daq_data_channel_flags="no"])
++have_daq_data_channel_flags="no"
+ AC_MSG_RESULT($have_daq_data_channel_flags)
+ if test "x$have_daq_data_channel_flags" = "xyes"; then
+     CCONFIGFLAGS="${CCONFIGFLAGS} -DHAVE_DAQ_DATA_CHANNEL_PARAMS"
+@@ -850,17 +766,7 @@ if test "x$have_daq_data_channel_flags" = "xyes"; then
+ fi
+ 
+ AC_MSG_CHECKING([for separate IP versions on pinhole endpoints])
+-AC_RUN_IFELSE(
+-[AC_LANG_PROGRAM(
+-[[
+-#include <daq.h>
+-]],
+-[[
+-   DAQ_DP_key_t dpKey;
+-   dpKey.src_af = 0;
+-]])],
+-[have_daq_data_channel_separate_ip_versions="yes"],
+-[have_daq_data_channel_separate_ip_versions="no"])
++have_daq_data_channel_separate_ip_versions="no"
+ AC_MSG_RESULT($have_daq_data_channel_separate_ip_versions)
+ if test "x$have_daq_data_channel_separate_ip_versions" = "xyes"; then
+     CCONFIGFLAGS="${CCONFIGFLAGS} -DHAVE_DAQ_DATA_CHANNEL_SEPARATE_IP_VERSIONS"
+@@ -869,17 +775,7 @@ if test "x$have_daq_data_channel_separate_ip_versions" = "xyes"; then
+ fi
+ 
+ AC_MSG_CHECKING([for DAQ_VERDICT_RETRY])
+-AC_RUN_IFELSE(
+-[AC_LANG_PROGRAM(
+-[[
+-#include <daq.h>
+-]],
+-[[
+-   DAQ_Verdict verdict;
+-   verdict = DAQ_VERDICT_RETRY;
+-]])],
+-[have_daq_verdict_retry="yes"],
+-[have_daq_verdict_retry="no"])
++have_daq_verdict_retry="yes"
+ AC_MSG_RESULT($have_daq_verdict_retry)
+ if test "x$have_daq_verdict_retry" = "xyes"; then
+     AC_DEFINE([HAVE_DAQ_VERDICT_RETRY],[1],
+@@ -887,17 +783,7 @@ if test "x$have_daq_verdict_retry" = "xyes"; then
+ fi
+ 
+ AC_MSG_CHECKING([for daq packet trace])
+-AC_RUN_IFELSE(
+-[AC_LANG_PROGRAM(
+-[[
+-#include <daq.h>
+-]],
+-[[
+-   DAQ_PktHdr_t hdr;
+-   hdr.flags = DAQ_PKT_FLAG_TRACE_ENABLED;
+-]])],
+-[have_daq_packet_trace="yes"],
+-[have_daq_packet_trace="no"])
++have_daq_packet_trace="yes"
+ AC_MSG_RESULT($have_daq_packet_trace)
+ if test "x$have_daq_packet_trace" = "xyes"; then
+     AC_DEFINE([HAVE_DAQ_PKT_TRACE],[1],
+@@ -907,17 +793,7 @@ else
+ fi
+ 
+ AC_MSG_CHECKING([for daq verdict reason])
+-AC_RUN_IFELSE(
+-[AC_LANG_PROGRAM(
+-[[
+-#include <daq.h>
+-]],
+-[[
+-   DAQ_ModFlow_t fl;
+-   fl.type = DAQ_MODFLOW_TYPE_VER_REASON;
+-]])],
+-[have_daq_verdict_reason="yes"],
+-[have_daq_verdict_reason="no"])
++have_daq_verdict_reason="yes"
+ AC_MSG_RESULT($have_daq_verdict_reason)
+ if test "x$have_daq_verdict_reason" = "xyes"; then
+     AC_DEFINE([HAVE_DAQ_VERDICT_REASON],[1],
+-- 
+2.7.4
+
diff --git a/meta-networking/recipes-connectivity/snort/snort_2.9.7.5.bb b/meta-networking/recipes-connectivity/snort/snort_2.9.11.1.bb
similarity index 89%
rename from meta-networking/recipes-connectivity/snort/snort_2.9.7.5.bb
rename to meta-networking/recipes-connectivity/snort/snort_2.9.11.1.bb
index 24b91b9..fdf41bf 100644
--- a/meta-networking/recipes-connectivity/snort/snort_2.9.7.5.bb
+++ b/meta-networking/recipes-connectivity/snort/snort_2.9.11.1.bb
@@ -6,19 +6,15 @@ LIC_FILES_CHKSUM = "file://COPYING;md5=78fa8ef966b48fbf9095e13cc92377c5"
 
 DEPENDS = "xz libpcap libpcre daq libdnet util-linux daq-native libtirpc"
 
-SRC_URI = " ${GENTOO_MIRROR}/${BP}.tar.gz;name=tarball \
+SRC_URI = "https://www.snort.org/downloads/snort/${BP}.tar.gz \
     file://snort.init \
-    file://disable-inaddr-none.patch \
-    file://disable-dap-address-space-id.patch \
-    file://disable-daq-flow-id.patch \
-    file://disable-daq-verdict-retry.patch \
     file://0001-libpcap-search-sysroot-for-headers.patch \
-    file://0001-fix-do_package-failed-since-snort-2.9.7.0.patch \
     file://fix-host-contamination-when-enable-static-daq.patch \
+    file://disable-run-test-program-while-cross-compiling.patch \
 "
 
-SRC_URI[tarball.md5sum] = "fd271788c0f8876be87a858a9142f202"
-SRC_URI[tarball.sha256sum] = "ad03f11b5301b16642199a86aa90388eaa53f5003f83b0c5595745a490047be1"
+SRC_URI[md5sum] = "378e3938b2b5c8e358f942d0ffce18cc"
+SRC_URI[sha256sum] = "9f6b3aeac5a109f55504bd370564ac431cb1773507929dc461626898f33f46cd"
 
 UPSTREAM_CHECK_URI = "https://www.snort.org/downloads"
 UPSTREAM_CHECK_REGEX = "snort-(?P<pver>\d+(\.\d+)+)\.tar"
-- 
2.7.4



^ permalink raw reply related

* Re: [PATCH] x86/entry/64: Remove %ebx handling from error_entry/exit
From: Andy Lutomirski @ 2018-07-24  2:31 UTC (permalink / raw)
  To: Greg KH
  Cc: Andy Lutomirski, X86 ML, LKML, Borislav Petkov, Linus Torvalds,
	Dave Hansen, Brian Gerst, Dominik Brodowski, Ingo Molnar,
	H. Peter Anvin, Thomas Gleixner, Boris Ostrovsky, Juergen Gross,
	xen-devel, stable
In-Reply-To: <20180723072505.GA24222@kroah.com>

On Mon, Jul 23, 2018 at 12:25 AM, Greg KH <gregkh@linuxfoundation.org> wrote:
> On Sun, Jul 22, 2018 at 11:05:09AM -0700, Andy Lutomirski wrote:
>> error_entry and error_exit communicate the user vs kernel status of
>> the frame using %ebx.  This is unnecessary -- the information is in
>> regs->cs.  Just use regs->cs.
>>
>> This makes error_entry simpler and makes error_exit more robust.
>>
>> It also fixes a nasty bug.  Before all the Spectre nonsense, The
>> xen_failsafe_callback entry point returned like this:
>>
>>         ALLOC_PT_GPREGS_ON_STACK
>>         SAVE_C_REGS
>>         SAVE_EXTRA_REGS
>>         ENCODE_FRAME_POINTER
>>         jmp     error_exit
>>
>> And it did not go through error_entry.  This was bogus: RBX
>> contained garbage, and error_exit expected a flag in RBX.
>> Fortunately, it generally contained *nonzero* garbage, so the
>> correct code path was used.  As part of the Spectre fixes, code was
>> added to clear RBX to mitigate certain speculation attacks.  Now,
>> depending on kernel configuration, RBX got zeroed and, when running
>> some Wine workloads, the kernel crashes.  This was introduced by:
>>
>>     commit 3ac6d8c787b8 ("x86/entry/64: Clear registers for
>>     exceptions/interrupts, to reduce speculation attack surface")
>>
>> With this patch applied, RBX is no longer needed as a flag, and the
>> problem goes away.
>>
>> I suspect that malicious userspace could use this bug to crash the
>> kernel even without the offending patch applied, though.
>>
>> [Historical note: I wrote this patch as a cleanup before I was aware
>>  of the bug it fixed.]
>>
>> [Note to stable maintainers: this should probably get applied to all
>>  kernels.  If you're nervous about that, a more conservative fix to
>>  add xorl %ebx,%ebx; incl %ebx before the jump to error_exit should
>>  also fix the problem.]
>>
>> Cc: Brian Gerst <brgerst@gmail.com>
>> Cc: Borislav Petkov <bp@alien8.de>
>> Cc: Dominik Brodowski <linux@dominikbrodowski.net>
>> Cc: Ingo Molnar <mingo@redhat.com>
>> Cc: "H. Peter Anvin" <hpa@zytor.com>
>> Cc: Thomas Gleixner <tglx@linutronix.de>
>> Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com>
>> Cc: Juergen Gross <jgross@suse.com>
>> Cc: xen-devel@lists.xenproject.org
>> Cc: x86@kernel.org
>> Cc: stable@vger.kernel.org
>> Fixes: 3ac6d8c787b8 ("x86/entry/64: Clear registers for exceptions/interrupts, to reduce speculation attack surface")
>> Reported-and-tested-by: "M. Vefa Bicakci" <m.v.b@runbox.com>
>> Signed-off-by: Andy Lutomirski <luto@kernel.org>
>> ---
>>
>> I could also submit the conservative fix tagged for -stable and respin
>> this on top of it.  Ingo, Greg, what do you prefer?
>
> I don't care, this patch looks good to me to take as-is for the stable
> trees.  If you trust it in Linus's tree, it should be fine for others :)
>

My concern is more that something may work differently in older
kernels and there might be some subtle issue.  I'd be surprised, but
still.

^ permalink raw reply

* [PATCH] autotools.bbclass: fix autoreconf bbnote commandline arguments
From: Andre McCurdy @ 2018-07-24  2:32 UTC (permalink / raw)
  To: openembedded-core

Leaving -Wcross out of the bbnote version of the autoreconf command
seems to be a long standing inconsistency (dating back to the very
first commit in oe-core) but there's no obvious reason to do so.

Signed-off-by: Andre McCurdy <armccurdy@gmail.com>
---
 meta/classes/autotools.bbclass | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta/classes/autotools.bbclass b/meta/classes/autotools.bbclass
index cc857ac..81d3674 100644
--- a/meta/classes/autotools.bbclass
+++ b/meta/classes/autotools.bbclass
@@ -225,7 +225,7 @@ autotools_do_configure() {
 			find ${S} -ignore_readdir_race -name $i -delete
 		done
 
-		bbnote Executing ACLOCAL=\"$ACLOCAL\" autoreconf --verbose --install --force ${EXTRA_AUTORECONF} $acpaths
+		bbnote Executing ACLOCAL=\"$ACLOCAL\" autoreconf -Wcross --verbose --install --force ${EXTRA_AUTORECONF} $acpaths
 		ACLOCAL="$ACLOCAL" autoreconf -Wcross --verbose --install --force ${EXTRA_AUTORECONF} $acpaths || die "autoreconf execution failed."
 		cd $olddir
 	fi
-- 
1.9.1



^ permalink raw reply related

* Re: [PATCH] x86/entry/64: Remove %ebx handling from error_entry/exit
From: Andy Lutomirski @ 2018-07-24  2:31 UTC (permalink / raw)
  To: Greg KH
  Cc: Juergen Gross, X86 ML, Brian Gerst, Dave Hansen, LKML,
	Dominik Brodowski, Ingo Molnar, Borislav Petkov, stable,
	Andy Lutomirski, H. Peter Anvin, xen-devel, Thomas Gleixner,
	Linus Torvalds, Boris Ostrovsky
In-Reply-To: <20180723072505.GA24222@kroah.com>

On Mon, Jul 23, 2018 at 12:25 AM, Greg KH <gregkh@linuxfoundation.org> wrote:
> On Sun, Jul 22, 2018 at 11:05:09AM -0700, Andy Lutomirski wrote:
>> error_entry and error_exit communicate the user vs kernel status of
>> the frame using %ebx.  This is unnecessary -- the information is in
>> regs->cs.  Just use regs->cs.
>>
>> This makes error_entry simpler and makes error_exit more robust.
>>
>> It also fixes a nasty bug.  Before all the Spectre nonsense, The
>> xen_failsafe_callback entry point returned like this:
>>
>>         ALLOC_PT_GPREGS_ON_STACK
>>         SAVE_C_REGS
>>         SAVE_EXTRA_REGS
>>         ENCODE_FRAME_POINTER
>>         jmp     error_exit
>>
>> And it did not go through error_entry.  This was bogus: RBX
>> contained garbage, and error_exit expected a flag in RBX.
>> Fortunately, it generally contained *nonzero* garbage, so the
>> correct code path was used.  As part of the Spectre fixes, code was
>> added to clear RBX to mitigate certain speculation attacks.  Now,
>> depending on kernel configuration, RBX got zeroed and, when running
>> some Wine workloads, the kernel crashes.  This was introduced by:
>>
>>     commit 3ac6d8c787b8 ("x86/entry/64: Clear registers for
>>     exceptions/interrupts, to reduce speculation attack surface")
>>
>> With this patch applied, RBX is no longer needed as a flag, and the
>> problem goes away.
>>
>> I suspect that malicious userspace could use this bug to crash the
>> kernel even without the offending patch applied, though.
>>
>> [Historical note: I wrote this patch as a cleanup before I was aware
>>  of the bug it fixed.]
>>
>> [Note to stable maintainers: this should probably get applied to all
>>  kernels.  If you're nervous about that, a more conservative fix to
>>  add xorl %ebx,%ebx; incl %ebx before the jump to error_exit should
>>  also fix the problem.]
>>
>> Cc: Brian Gerst <brgerst@gmail.com>
>> Cc: Borislav Petkov <bp@alien8.de>
>> Cc: Dominik Brodowski <linux@dominikbrodowski.net>
>> Cc: Ingo Molnar <mingo@redhat.com>
>> Cc: "H. Peter Anvin" <hpa@zytor.com>
>> Cc: Thomas Gleixner <tglx@linutronix.de>
>> Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com>
>> Cc: Juergen Gross <jgross@suse.com>
>> Cc: xen-devel@lists.xenproject.org
>> Cc: x86@kernel.org
>> Cc: stable@vger.kernel.org
>> Fixes: 3ac6d8c787b8 ("x86/entry/64: Clear registers for exceptions/interrupts, to reduce speculation attack surface")
>> Reported-and-tested-by: "M. Vefa Bicakci" <m.v.b@runbox.com>
>> Signed-off-by: Andy Lutomirski <luto@kernel.org>
>> ---
>>
>> I could also submit the conservative fix tagged for -stable and respin
>> this on top of it.  Ingo, Greg, what do you prefer?
>
> I don't care, this patch looks good to me to take as-is for the stable
> trees.  If you trust it in Linus's tree, it should be fine for others :)
>

My concern is more that something may work differently in older
kernels and there might be some subtle issue.  I'd be surprised, but
still.

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xenproject.org
https://lists.xenproject.org/mailman/listinfo/xen-devel

^ permalink raw reply

* [PATCH] xfs_db: add -d to short help for write command
From: Eric Sandeen @ 2018-07-24  1:25 UTC (permalink / raw)
  To: linux-xfs

And note in the man page that -c and -d are exclusive.

Signed-off-by: Eric Sandeen <sandeen@redhat.com>
---

diff --git a/db/write.c b/db/write.c
index 5ef76bcd..bbaa609d 100644
--- a/db/write.c
+++ b/db/write.c
@@ -38,7 +38,7 @@ static int	write_f(int argc, char **argv);
 static void     write_help(void);
 
 static const cmdinfo_t	write_cmd =
-	{ "write", NULL, write_f, 0, -1, 0, N_("[-c] [field or value]..."),
+	{ "write", NULL, write_f, 0, -1, 0, N_("[-c|-d] [field or value]..."),
 	  N_("write value to disk"), write_help };
 
 void
diff --git a/man/man8/xfs_db.8 b/man/man8/xfs_db.8
index 10f2beb9..a1ee3514 100644
--- a/man/man8/xfs_db.8
+++ b/man/man8/xfs_db.8
@@ -837,7 +837,7 @@ and
 bits respectively, and their string equivalent reported
 (but no modifications are made).
 .TP
-.BI "write [\-c] [\-d] [" "field value" "] ..."
+.BI "write [\-c|\-d] [" "field value" "] ..."
 Write a value to disk.
 Specific fields can be set in structures (struct mode),
 or a block can be set to data values (data mode),


^ permalink raw reply related

* [PATCH 5/5 v2] toaster/orm/management/commands/lsupdates.py: Use new layerindexlib module
From: Mark Hatle @ 2018-07-24  2:29 UTC (permalink / raw)
  To: bitbake-devel
In-Reply-To: <20180724022914.185634-1-mark.hatle@windriver.com>

Change lsupdates.py to use the new layerindexlib module to load the data from
the public layer index.  It still does all of the manual parsing.  This
is intended to be a stop gap until the toaster can use the module itself to
manage the data.

Everything else is functionally equivalent to the prior version.

Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
---
 lib/toaster/orm/management/commands/lsupdates.py | 216 +++++++++--------------
 1 file changed, 87 insertions(+), 129 deletions(-)

diff --git a/lib/toaster/orm/management/commands/lsupdates.py b/lib/toaster/orm/management/commands/lsupdates.py
index efc6b3a..3f3de4f 100644
--- a/lib/toaster/orm/management/commands/lsupdates.py
+++ b/lib/toaster/orm/management/commands/lsupdates.py
@@ -29,14 +29,18 @@ from orm.models import ToasterSetting
 import os
 import sys
 
-import json
 import logging
 import threading
 import time
 logger = logging.getLogger("toaster")
 
+# At some point this needs to be made dynamic
 DEFAULT_LAYERINDEX_SERVER = "http://layers.openembedded.org/layerindex/api/"
 
+# load Bitbake components
+path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+sys.path.insert(0, path)
+
 
 class Spinner(threading.Thread):
     """ A simple progress spinner to indicate download/parsing is happening"""
@@ -86,45 +90,6 @@ class Command(BaseCommand):
             self.apiurl = ToasterSetting.objects.get(name = 'CUSTOM_LAYERINDEX_SERVER').value
 
         assert self.apiurl is not None
-        try:
-            from urllib.request import urlopen, URLError
-            from urllib.parse import urlparse
-        except ImportError:
-            from urllib2 import urlopen, URLError
-            from urlparse import urlparse
-
-        proxy_settings = os.environ.get("http_proxy", None)
-
-        def _get_json_response(apiurl=None):
-            if None == apiurl:
-                apiurl=self.apiurl
-            http_progress = Spinner()
-            http_progress.start()
-
-            _parsedurl = urlparse(apiurl)
-            path = _parsedurl.path
-
-            # logger.debug("Fetching %s", apiurl)
-            try:
-                res = urlopen(apiurl)
-            except URLError as e:
-                raise Exception("Failed to read %s: %s" % (path, e.reason))
-
-            parsed = json.loads(res.read().decode('utf-8'))
-
-            http_progress.stop()
-            return parsed
-
-        # verify we can get the basic api
-        try:
-            apilinks = _get_json_response()
-        except Exception as e:
-            import traceback
-            if proxy_settings is not None:
-                logger.info("EE: Using proxy %s" % proxy_settings)
-            logger.warning("EE: could not connect to %s, skipping update:"
-                           "%s\n%s" % (self.apiurl, e, traceback.format_exc()))
-            return
 
         # update branches; only those that we already have names listed in the
         # Releases table
@@ -133,112 +98,117 @@ class Command(BaseCommand):
         if len(whitelist_branch_names) == 0:
             raise Exception("Failed to make list of branches to fetch")
 
+        self.apiurl += ";branch=%s" % ','.join(whitelist_branch_names)
+
+        http_progress = Spinner()
+
         logger.info("Fetching metadata releases for %s",
                     " ".join(whitelist_branch_names))
 
-        branches_info = _get_json_response(apilinks['branches'] +
-                                           "?filter=name:%s"
-                                           % "OR".join(whitelist_branch_names))
+
+        import layerindexlib
+
+        layerindex = layerindexlib.LayerIndex({"DUMMY" : "VALUE"})
+
+        http_progress.start()
+        layerindex.load_layerindex(self.apiurl)
+        http_progress.stop()
+
+        # We know we're only processing one entry, so we reference it here
+        # (this is cheating...)
+        lindex = layerindex.lindex[0]
 
         # Map the layer index branches to toaster releases
         li_branch_id_to_toaster_release = {}
 
-        total = len(branches_info)
-        for i, branch in enumerate(branches_info):
-            li_branch_id_to_toaster_release[branch['id']] = \
-                    Release.objects.get(name=branch['name'])
+        logger.info("Processing branches")
+
+        total = len(lindex.branches)
+        for i, branchId in enumerate(lindex.branches):
+            li_branch_id_to_toaster_release[branchId] = \
+                    Release.objects.get(name=lindex.branches[branchId].name)
             self.mini_progress("Releases", i, total)
 
         # keep a track of the layerindex (li) id mappings so that
         # layer_versions can be created for these layers later on
         li_layer_id_to_toaster_layer_id = {}
 
-        logger.info("Fetching layers")
-
-        layers_info = _get_json_response(apilinks['layerItems'])
+        logger.info("Processing layers")
 
-        total = len(layers_info)
-        for i, li in enumerate(layers_info):
+        total = len(lindex.layerItems)
+        for i, liId in enumerate(lindex.layerItems):
             try:
-                l, created = Layer.objects.get_or_create(name=li['name'])
-                l.up_date = li['updated']
-                l.summary = li['summary']
-                l.description = li['description']
+                l, created = Layer.objects.get_or_create(name=lindex.layerItems[liId].name)
+                l.up_date = lindex.layerItems[liId].updated
+                l.summary = lindex.layerItems[liId].summary
+                l.description = lindex.layerItems[liId].description
 
                 if created:
                     # predefined layers in the fixtures (for example poky.xml)
                     # always preempt the Layer Index for these values
-                    l.vcs_url = li['vcs_url']
-                    l.vcs_web_url = li['vcs_web_url']
-                    l.vcs_web_tree_base_url = li['vcs_web_tree_base_url']
-                    l.vcs_web_file_base_url = li['vcs_web_file_base_url']
+                    l.vcs_url = lindex.layerItems[liId].vcs_url
+                    l.vcs_web_url = lindex.layerItems[liId].vcs_web_url
+                    l.vcs_web_tree_base_url = lindex.layerItems[liId].vcs_web_tree_base_url
+                    l.vcs_web_file_base_url = lindex.layerItems[liId].vcs_web_file_base_url
                 l.save()
             except Layer.MultipleObjectsReturned:
                 logger.info("Skipped %s as we found multiple layers and "
                             "don't know which to update" %
                             li['name'])
 
-            li_layer_id_to_toaster_layer_id[li['id']] = l.pk
+            li_layer_id_to_toaster_layer_id[liId] = l.pk
 
             self.mini_progress("layers", i, total)
 
         # update layer_versions
-        logger.info("Fetching layer versions")
-        layerbranches_info = _get_json_response(
-            apilinks['layerBranches'] + "?filter=branch__name:%s" %
-            "OR".join(whitelist_branch_names))
+        logger.info("Processing layer branches")
 
         # Map Layer index layer_branch object id to
         # layer_version toaster object id
         li_layer_branch_id_to_toaster_lv_id = {}
 
-        total = len(layerbranches_info)
-        for i, lbi in enumerate(layerbranches_info):
+        total = len(lindex.layerBranches)
+        for i, lbiId in enumerate(lindex.layerBranches):
             # release as defined by toaster map to layerindex branch
-            release = li_branch_id_to_toaster_release[lbi['branch']]
+            release = li_branch_id_to_toaster_release[lindex.layerBranches[lbiId].branch_id]
 
             try:
                 lv, created = Layer_Version.objects.get_or_create(
                     layer=Layer.objects.get(
-                        pk=li_layer_id_to_toaster_layer_id[lbi['layer']]),
+                        pk=li_layer_id_to_toaster_layer_id[lindex.layerBranches[lbiId].layer_id]),
                     release=release
                 )
             except KeyError:
                 logger.warning(
                     "No such layerindex layer referenced by layerbranch %d" %
-                    lbi['layer'])
+                    lindex.layerBranches[lbiId].layer_id)
                 continue
 
             if created:
-                lv.release = li_branch_id_to_toaster_release[lbi['branch']]
-                lv.up_date = lbi['updated']
-                lv.commit = lbi['actual_branch']
-                lv.dirpath = lbi['vcs_subdir']
+                lv.release = li_branch_id_to_toaster_release[lindex.layerBranches[lbiId].branch_id]
+                lv.up_date = lindex.layerBranches[lbiId].updated
+                lv.commit = lindex.layerBranches[lbiId].actual_branch
+                lv.dirpath = lindex.layerBranches[lbiId].vcs_subdir
                 lv.save()
 
-            li_layer_branch_id_to_toaster_lv_id[lbi['id']] =\
+            li_layer_branch_id_to_toaster_lv_id[lindex.layerBranches[lbiId].id] =\
                 lv.pk
             self.mini_progress("layer versions", i, total)
 
-        logger.info("Fetching layer version dependencies")
-        # update layer dependencies
-        layerdependencies_info = _get_json_response(
-            apilinks['layerDependencies'] +
-            "?filter=layerbranch__branch__name:%s" %
-            "OR".join(whitelist_branch_names))
+        logger.info("Processing layer dependencies")
 
         dependlist = {}
-        for ldi in layerdependencies_info:
+        for ldiId in lindex.layerDependencies:
             try:
                 lv = Layer_Version.objects.get(
-                    pk=li_layer_branch_id_to_toaster_lv_id[ldi['layerbranch']])
+                    pk=li_layer_branch_id_to_toaster_lv_id[lindex.layerDependencies[ldiId].layerbranch_id])
             except Layer_Version.DoesNotExist as e:
                 continue
 
             if lv not in dependlist:
                 dependlist[lv] = []
             try:
-                layer_id = li_layer_id_to_toaster_layer_id[ldi['dependency']]
+                layer_id = li_layer_id_to_toaster_layer_id[lindex.layerDependencies[ldiId].dependency_id]
 
                 dependlist[lv].append(
                     Layer_Version.objects.get(layer__pk=layer_id,
@@ -247,7 +217,7 @@ class Command(BaseCommand):
             except Layer_Version.DoesNotExist:
                 logger.warning("Cannot find layer version (ls:%s),"
                                "up_id:%s lv:%s" %
-                               (self, ldi['dependency'], lv))
+                               (self, lindex.layerDependencies[ldiId].dependency_id, lv))
 
         total = len(dependlist)
         for i, lv in enumerate(dependlist):
@@ -258,73 +228,61 @@ class Command(BaseCommand):
             self.mini_progress("Layer version dependencies", i, total)
 
         # update Distros
-        logger.info("Fetching distro information")
-        distros_info = _get_json_response(
-            apilinks['distros'] + "?filter=layerbranch__branch__name:%s" %
-            "OR".join(whitelist_branch_names))
+        logger.info("Processing distro information")
 
-        total = len(distros_info)
-        for i, di in enumerate(distros_info):
+        total = len(lindex.distros)
+        for i, diId in enumerate(lindex.distros):
             distro, created = Distro.objects.get_or_create(
-                name=di['name'],
+                name=lindex.distros[diId].name,
                 layer_version=Layer_Version.objects.get(
-                    pk=li_layer_branch_id_to_toaster_lv_id[di['layerbranch']]))
-            distro.up_date = di['updated']
-            distro.name = di['name']
-            distro.description = di['description']
+                    pk=li_layer_branch_id_to_toaster_lv_id[lindex.distros[diId].layerbranch_id]))
+            distro.up_date = lindex.distros[diId].updated
+            distro.name = lindex.distros[diId].name
+            distro.description = lindex.distros[diId].description
             distro.save()
             self.mini_progress("distros", i, total)
 
         # update machines
-        logger.info("Fetching machine information")
-        machines_info = _get_json_response(
-            apilinks['machines'] + "?filter=layerbranch__branch__name:%s" %
-            "OR".join(whitelist_branch_names))
+        logger.info("Processing machine information")
 
-        total = len(machines_info)
-        for i, mi in enumerate(machines_info):
+        total = len(lindex.machines)
+        for i, miId in enumerate(lindex.machines):
             mo, created = Machine.objects.get_or_create(
-                name=mi['name'],
+                name=lindex.machines[miId].name,
                 layer_version=Layer_Version.objects.get(
-                    pk=li_layer_branch_id_to_toaster_lv_id[mi['layerbranch']]))
-            mo.up_date = mi['updated']
-            mo.name = mi['name']
-            mo.description = mi['description']
+                    pk=li_layer_branch_id_to_toaster_lv_id[lindex.machines[miId].layerbranch_id]))
+            mo.up_date = lindex.machines[miId].updated
+            mo.name = lindex.machines[miId].name
+            mo.description = lindex.machines[miId].description
             mo.save()
             self.mini_progress("machines", i, total)
 
         # update recipes; paginate by layer version / layer branch
-        logger.info("Fetching recipe information")
-        recipes_info = _get_json_response(
-            apilinks['recipes'] + "?filter=layerbranch__branch__name:%s" %
-            "OR".join(whitelist_branch_names))
+        logger.info("Processing recipe information")
 
-        total = len(recipes_info)
-        for i, ri in enumerate(recipes_info):
+        total = len(lindex.recipes)
+        for i, riId in enumerate(lindex.recipes):
             try:
-                lv_id = li_layer_branch_id_to_toaster_lv_id[ri['layerbranch']]
+                lv_id = li_layer_branch_id_to_toaster_lv_id[lindex.recipes[riId].layerbranch_id]
                 lv = Layer_Version.objects.get(pk=lv_id)
 
                 ro, created = Recipe.objects.get_or_create(
                     layer_version=lv,
-                    name=ri['pn']
+                    name=lindex.recipes[riId].pn
                 )
 
                 ro.layer_version = lv
-                ro.up_date = ri['updated']
-                ro.name = ri['pn']
-                ro.version = ri['pv']
-                ro.summary = ri['summary']
-                ro.description = ri['description']
-                ro.section = ri['section']
-                ro.license = ri['license']
-                ro.homepage = ri['homepage']
-                ro.bugtracker = ri['bugtracker']
-                ro.file_path = ri['filepath'] + "/" + ri['filename']
-                if 'inherits' in ri:
-                    ro.is_image = 'image' in ri['inherits'].split()
-                else:  # workaround for old style layer index
-                    ro.is_image = "-image-" in ri['pn']
+                ro.up_date = lindex.recipes[riId].updated
+                ro.name = lindex.recipes[riId].pn
+                ro.version = lindex.recipes[riId].pv
+                ro.summary = lindex.recipes[riId].summary
+                ro.description = lindex.recipes[riId].description
+                ro.section = lindex.recipes[riId].section
+                ro.license = lindex.recipes[riId].license
+                ro.homepage = lindex.recipes[riId].homepage
+                ro.bugtracker = lindex.recipes[riId].bugtracker
+                ro.file_path = lindex.recipes[riId].fullpath
+                ro.is_image = 'image' in lindex.recipes[riId].inherits.split()
                 ro.save()
             except Exception as e:
                 logger.warning("Failed saving recipe %s", e)
-- 
1.8.3.1



^ permalink raw reply related

* [PATCH 4/5] bitbake-layers: disable parsing for layerindex commands
From: Mark Hatle @ 2018-07-24  2:29 UTC (permalink / raw)
  To: bitbake-devel
In-Reply-To: <20180724022914.185634-1-mark.hatle@windriver.com>

These don't need to access recipe information, so let's not waste the
user's time parsing all recipes.

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
---
 lib/bblayers/layerindex.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/bblayers/layerindex.py b/lib/bblayers/layerindex.py
index 121c7d5..9f02a9d 100644
--- a/lib/bblayers/layerindex.py
+++ b/lib/bblayers/layerindex.py
@@ -197,12 +197,12 @@ class LayerIndexPlugin(ActionPlugin):
         self.do_layerindex_fetch(args)
 
     def register_commands(self, sp):
-        parser_layerindex_fetch = self.add_command(sp, 'layerindex-fetch', self.do_layerindex_fetch)
+        parser_layerindex_fetch = self.add_command(sp, 'layerindex-fetch', self.do_layerindex_fetch, parserecipes=False)
         parser_layerindex_fetch.add_argument('-n', '--show-only', help='show dependencies and do nothing else', action='store_true')
         parser_layerindex_fetch.add_argument('-b', '--branch', help='branch name to fetch')
         parser_layerindex_fetch.add_argument('-i', '--ignore', help='assume the specified layers do not need to be fetched/added (separate multiple layers with commas, no spaces)', metavar='LAYER')
         parser_layerindex_fetch.add_argument('layername', nargs='+', help='layer to fetch')
 
-        parser_layerindex_show_depends = self.add_command(sp, 'layerindex-show-depends', self.do_layerindex_show_depends)
+        parser_layerindex_show_depends = self.add_command(sp, 'layerindex-show-depends', self.do_layerindex_show_depends, parserecipes=False)
         parser_layerindex_show_depends.add_argument('-b', '--branch', help='branch name to fetch')
         parser_layerindex_show_depends.add_argument('layername', nargs='+', help='layer to query')
-- 
1.8.3.1



^ permalink raw reply related

* [PATCH 3/5 v2] bblayers/layerindex.py: Switch to use the new layerindexlib class
From: Mark Hatle @ 2018-07-24  2:29 UTC (permalink / raw)
  To: bitbake-devel
In-Reply-To: <20180724022914.185634-1-mark.hatle@windriver.com>

Display changes:
The output will now include references to the layers that the user already
has on their system.  It does this by querying the cooker derived index.

The code that enables this behavior is labeled as 'TODO' currently.  As
part of the work we need to make a final determination if this is the
desired output.

Also changed the default branch to no longer define itself as 'master'.

When the user does NOT set a branch, the default is now the
'LAYERSERIES_CORENAMES', and if that doesn't exist 'master'.  This is
subtly different in behavior, but more consistent with user expectations.

Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
---
 lib/bblayers/layerindex.py | 304 ++++++++++++++++++---------------------------
 1 file changed, 119 insertions(+), 185 deletions(-)

diff --git a/lib/bblayers/layerindex.py b/lib/bblayers/layerindex.py
index 53c858d..121c7d5 100644
--- a/lib/bblayers/layerindex.py
+++ b/lib/bblayers/layerindex.py
@@ -1,10 +1,9 @@
+import layerindexlib
+
 import argparse
-import http.client
-import json
 import logging
 import os
 import subprocess
-import urllib.parse
 
 from bblayers.action import ActionPlugin
 
@@ -21,110 +20,6 @@ class LayerIndexPlugin(ActionPlugin):
     This class inherits ActionPlugin to get do_add_layer.
     """
 
-    def get_json_data(self, apiurl):
-        proxy_settings = os.environ.get("http_proxy", None)
-        conn = None
-        _parsedurl = urllib.parse.urlparse(apiurl)
-        path = _parsedurl.path
-        query = _parsedurl.query
-
-        def parse_url(url):
-            parsedurl = urllib.parse.urlparse(url)
-            if parsedurl.netloc[0] == '[':
-                host, port = parsedurl.netloc[1:].split(']', 1)
-                if ':' in port:
-                    port = port.rsplit(':', 1)[1]
-                else:
-                    port = None
-            else:
-                if parsedurl.netloc.count(':') == 1:
-                    (host, port) = parsedurl.netloc.split(":")
-                else:
-                    host = parsedurl.netloc
-                    port = None
-            return (host, 80 if port is None else int(port))
-
-        if proxy_settings is None:
-            host, port = parse_url(apiurl)
-            conn = http.client.HTTPConnection(host, port)
-            conn.request("GET", path + "?" + query)
-        else:
-            host, port = parse_url(proxy_settings)
-            conn = http.client.HTTPConnection(host, port)
-            conn.request("GET", apiurl)
-
-        r = conn.getresponse()
-        if r.status != 200:
-            raise Exception("Failed to read " + path + ": %d %s" % (r.status, r.reason))
-        return json.loads(r.read().decode())
-
-    def get_layer_deps(self, layername, layeritems, layerbranches, layerdependencies, branchnum, selfname=False):
-        def layeritems_info_id(items_name, layeritems):
-            litems_id = None
-            for li in layeritems:
-                if li['name'] == items_name:
-                    litems_id = li['id']
-                    break
-            return litems_id
-
-        def layerbranches_info(items_id, layerbranches):
-            lbranch = {}
-            for lb in layerbranches:
-                if lb['layer'] == items_id and lb['branch'] == branchnum:
-                    lbranch['id'] = lb['id']
-                    lbranch['vcs_subdir'] = lb['vcs_subdir']
-                    break
-            return lbranch
-
-        def layerdependencies_info(lb_id, layerdependencies):
-            ld_deps = []
-            for ld in layerdependencies:
-                if ld['layerbranch'] == lb_id and not ld['dependency'] in ld_deps:
-                    ld_deps.append(ld['dependency'])
-            if not ld_deps:
-                logger.error("The dependency of layerDependencies is not found.")
-            return ld_deps
-
-        def layeritems_info_name_subdir(items_id, layeritems):
-            litems = {}
-            for li in layeritems:
-                if li['id'] == items_id:
-                    litems['vcs_url'] = li['vcs_url']
-                    litems['name'] = li['name']
-                    break
-            return litems
-
-        if selfname:
-            selfid = layeritems_info_id(layername, layeritems)
-            lbinfo = layerbranches_info(selfid, layerbranches)
-            if lbinfo:
-                selfsubdir = lbinfo['vcs_subdir']
-            else:
-                logger.error("%s is not found in the specified branch" % layername)
-                return
-            selfurl = layeritems_info_name_subdir(selfid, layeritems)['vcs_url']
-            if selfurl:
-                return selfurl, selfsubdir
-            else:
-                logger.error("Cannot get layer %s git repo and subdir" % layername)
-                return
-        ldict = {}
-        itemsid = layeritems_info_id(layername, layeritems)
-        if not itemsid:
-            return layername, None
-        lbid = layerbranches_info(itemsid, layerbranches)
-        if lbid:
-            lbid = layerbranches_info(itemsid, layerbranches)['id']
-        else:
-            logger.error("%s is not found in the specified branch" % layername)
-            return None, None
-        for dependency in layerdependencies_info(lbid, layerdependencies):
-            lname = layeritems_info_name_subdir(dependency, layeritems)['name']
-            lurl = layeritems_info_name_subdir(dependency, layeritems)['vcs_url']
-            lsubdir = layerbranches_info(dependency, layerbranches)['vcs_subdir']
-            ldict[lname] = lurl, lsubdir
-        return None, ldict
-
     def get_fetch_layer(self, fetchdir, url, subdir, fetch_layer):
         layername = self.get_layer_name(url)
         if os.path.splitext(layername)[1] == '.git':
@@ -136,95 +31,124 @@ class LayerIndexPlugin(ActionPlugin):
                 result = subprocess.call('git clone %s %s' % (url, repodir), shell = True)
                 if result:
                     logger.error("Failed to download %s" % url)
-                    return None, None
+                    return None, None, None
                 else:
-                    return layername, layerdir
+                    return subdir, layername, layerdir
             else:
                 logger.plain("Repository %s needs to be fetched" % url)
-                return layername, layerdir
+                return subdir, layername, layerdir
         elif os.path.exists(layerdir):
-            return layername, layerdir
+            return subdir, layername, layerdir
         else:
             logger.error("%s is not in %s" % (url, subdir))
-        return None, None
+        return None, None, None
 
     def do_layerindex_fetch(self, args):
         """Fetches a layer from a layer index along with its dependent layers, and adds them to conf/bblayers.conf.
 """
-        apiurl = self.tinfoil.config_data.getVar('BBLAYERS_LAYERINDEX_URL')
-        if not apiurl:
-            logger.error("Cannot get BBLAYERS_LAYERINDEX_URL")
-            return 1
+
+        def _construct_url(baseurls, branches):
+            urls = []
+            for baseurl in baseurls:
+                if baseurl[-1] != '/':
+                    baseurl += '/'
+
+                if not baseurl.startswith('cooker'):
+                    baseurl += "api/"
+
+                if branches:
+                    baseurl += ";branch=%s" % ','.join(branches)
+
+                urls.append(baseurl)
+
+            return urls
+
+
+        # Set the default...
+        if args.branch:
+            branches = [args.branch]
         else:
-            if apiurl[-1] != '/':
-                apiurl += '/'
-            apiurl += "api/"
-        apilinks = self.get_json_data(apiurl)
-        branches = self.get_json_data(apilinks['branches'])
-
-        branchnum = 0
-        for branch in branches:
-            if branch['name'] == args.branch:
-                branchnum = branch['id']
-                break
-        if branchnum == 0:
-            validbranches = ', '.join([branch['name'] for branch in branches])
-            logger.error('Invalid layer branch name "%s". Valid branches: %s' % (args.branch, validbranches))
-            return 1
+            branches = (self.tinfoil.config_data.getVar('LAYERSERIES_CORENAMES') or 'master').split()
+        logger.debug(1, 'Trying branches: %s' % branches)
 
         ignore_layers = []
-        for collection in self.tinfoil.config_data.getVar('BBFILE_COLLECTIONS').split():
-            lname = self.tinfoil.config_data.getVar('BBLAYERS_LAYERINDEX_NAME_%s' % collection)
-            if lname:
-                ignore_layers.append(lname)
-
         if args.ignore:
             ignore_layers.extend(args.ignore.split(','))
 
-        layeritems = self.get_json_data(apilinks['layerItems'])
-        layerbranches = self.get_json_data(apilinks['layerBranches'])
-        layerdependencies = self.get_json_data(apilinks['layerDependencies'])
-        invaluenames = []
-        repourls = {}
-        printlayers = []
-
-        def query_dependencies(layers, layeritems, layerbranches, layerdependencies, branchnum):
-            depslayer = []
-            for layername in layers:
-                invaluename, layerdict = self.get_layer_deps(layername, layeritems, layerbranches, layerdependencies, branchnum)
-                if layerdict:
-                    repourls[layername] = self.get_layer_deps(layername, layeritems, layerbranches, layerdependencies, branchnum, selfname=True)
-                    for layer in layerdict:
-                        if not layer in ignore_layers:
-                            depslayer.append(layer)
-                        printlayers.append((layername, layer, layerdict[layer][0], layerdict[layer][1]))
-                        if not layer in ignore_layers and not layer in repourls:
-                            repourls[layer] = (layerdict[layer][0], layerdict[layer][1])
-                if invaluename and not invaluename in invaluenames:
-                    invaluenames.append(invaluename)
-            return depslayer
-
-        depslayers = query_dependencies(args.layername, layeritems, layerbranches, layerdependencies, branchnum)
-        while depslayers:
-            depslayer = query_dependencies(depslayers, layeritems, layerbranches, layerdependencies, branchnum)
-            depslayers = depslayer
-        if invaluenames:
-            for invaluename in invaluenames:
-                logger.error('Layer "%s" not found in layer index' % invaluename)
-            return 1
-        logger.plain("%s  %s  %s  %s" % ("Layer".ljust(19), "Required by".ljust(19), "Git repository".ljust(54), "Subdirectory"))
-        logger.plain('=' * 115)
-        for layername in args.layername:
-            layerurl = repourls[layername]
-            logger.plain("%s %s %s %s" % (layername.ljust(20), '-'.ljust(20), layerurl[0].ljust(55), layerurl[1]))
-        printedlayers = []
-        for layer, dependency, gitrepo, subdirectory in printlayers:
-            if dependency in printedlayers:
-                continue
-            logger.plain("%s %s %s %s" % (dependency.ljust(20), layer.ljust(20), gitrepo.ljust(55), subdirectory))
-            printedlayers.append(dependency)
-
-        if repourls:
+        # Load the cooker DB
+        cookerIndex = layerindexlib.LayerIndex(self.tinfoil.config_data)
+        cookerIndex.load_layerindex('cooker://', load='layerDependencies')
+
+        # Fast path, check if we already have what has been requested!
+        (dependencies, invalidnames) = cookerIndex.find_dependencies(names=args.layername, ignores=ignore_layers)
+        if not args.show_only and not invalidnames:
+            logger.plain("You already have the requested layer(s): %s" % args.layername)
+            return 0
+
+        # The information to show is already in the cookerIndex
+        if invalidnames:
+            # General URL to use to access the layer index
+            # While there is ONE right now, we're expect users could enter several
+            apiurl = self.tinfoil.config_data.getVar('BBLAYERS_LAYERINDEX_URL').split()
+            if not apiurl:
+                logger.error("Cannot get BBLAYERS_LAYERINDEX_URL")
+                return 1
+
+            remoteIndex = layerindexlib.LayerIndex(self.tinfoil.config_data)
+
+            for remoteurl in _construct_url(apiurl, branches):
+                logger.plain("Loading %s..." % remoteurl)
+                remoteIndex.load_layerindex(remoteurl)
+
+            if remoteIndex.is_empty():
+                logger.error("Remote layer index %s is empty for branches %s" % (apiurl, branches))
+                return 1
+
+            lIndex = cookerIndex + remoteIndex
+
+            (dependencies, invalidnames) = lIndex.find_dependencies(names=args.layername, ignores=ignore_layers)
+
+            if invalidnames:
+                for invaluename in invalidnames:
+                    logger.error('Layer "%s" not found in layer index' % invaluename)
+                return 1
+
+        logger.plain("%s  %s  %s" % ("Layer".ljust(49), "Git repository (branch)".ljust(54), "Subdirectory"))
+        logger.plain('=' * 125)
+
+        for deplayerbranch in dependencies:
+            layerBranch = dependencies[deplayerbranch][0]
+
+            # TODO: Determine display behavior
+            # This is the local content, uncomment to hide local
+            # layers from the display.
+            #if layerBranch.index.config['TYPE'] == 'cooker':
+            #    continue
+
+            layerDeps = dependencies[deplayerbranch][1:]
+
+            requiredby = []
+            recommendedby = []
+            for dep in layerDeps:
+                if dep.required:
+                    requiredby.append(dep.layer.name)
+                else:
+                    recommendedby.append(dep.layer.name)
+
+            logger.plain('%s %s %s' % (("%s:%s:%s" %
+                                  (layerBranch.index.config['DESCRIPTION'],
+                                  layerBranch.branch.name,
+                                  layerBranch.layer.name)).ljust(50),
+                                  ("%s (%s)" % (layerBranch.layer.vcs_url,
+                                  layerBranch.actual_branch)).ljust(55),
+                                  layerBranch.vcs_subdir
+                                               ))
+            if requiredby:
+                logger.plain('  required by: %s' % ' '.join(requiredby))
+            if recommendedby:
+                logger.plain('  recommended by: %s' % ' '.join(recommendedby))
+
+        if dependencies:
             fetchdir = self.tinfoil.config_data.getVar('BBLAYERS_FETCH_DIR')
             if not fetchdir:
                 logger.error("Cannot get BBLAYERS_FETCH_DIR")
@@ -232,8 +156,18 @@ class LayerIndexPlugin(ActionPlugin):
             if not os.path.exists(fetchdir):
                 os.makedirs(fetchdir)
             addlayers = []
-            for repourl, subdir in repourls.values():
-                name, layerdir = self.get_fetch_layer(fetchdir, repourl, subdir, not args.show_only)
+
+            for deplayerbranch in dependencies:
+                layerBranch = dependencies[deplayerbranch][0]
+
+                if layerBranch.index.config['TYPE'] == 'cooker':
+                    # Anything loaded via cooker is already local, skip it
+                    continue
+
+                subdir, name, layerdir = self.get_fetch_layer(fetchdir,
+                                                      layerBranch.layer.vcs_url,
+                                                      layerBranch.vcs_subdir,
+                                                      not args.show_only)
                 if not name:
                     # Error already shown
                     return 1
@@ -242,7 +176,7 @@ class LayerIndexPlugin(ActionPlugin):
             localargs = argparse.Namespace()
             localargs.layerdir = []
             localargs.force = args.force
-            for subdir, name, layerdir in set(addlayers):
+            for subdir, name, layerdir in addlayers:
                 if os.path.exists(layerdir):
                     if subdir:
                         logger.plain("Adding layer \"%s\" (%s) to conf/bblayers.conf" % (subdir, layerdir))
@@ -265,10 +199,10 @@ class LayerIndexPlugin(ActionPlugin):
     def register_commands(self, sp):
         parser_layerindex_fetch = self.add_command(sp, 'layerindex-fetch', self.do_layerindex_fetch)
         parser_layerindex_fetch.add_argument('-n', '--show-only', help='show dependencies and do nothing else', action='store_true')
-        parser_layerindex_fetch.add_argument('-b', '--branch', help='branch name to fetch (default %(default)s)', default='master')
+        parser_layerindex_fetch.add_argument('-b', '--branch', help='branch name to fetch')
         parser_layerindex_fetch.add_argument('-i', '--ignore', help='assume the specified layers do not need to be fetched/added (separate multiple layers with commas, no spaces)', metavar='LAYER')
         parser_layerindex_fetch.add_argument('layername', nargs='+', help='layer to fetch')
 
         parser_layerindex_show_depends = self.add_command(sp, 'layerindex-show-depends', self.do_layerindex_show_depends)
-        parser_layerindex_show_depends.add_argument('-b', '--branch', help='branch name to fetch (default %(default)s)', default='master')
+        parser_layerindex_show_depends.add_argument('-b', '--branch', help='branch name to fetch')
         parser_layerindex_show_depends.add_argument('layername', nargs='+', help='layer to query')
-- 
1.8.3.1



^ permalink raw reply related

* [PATCH 2/5 v2] layerindexlib: Initial layer index processing module implementation
From: Mark Hatle @ 2018-07-24  2:29 UTC (permalink / raw)
  To: bitbake-devel
In-Reply-To: <20180724022914.185634-1-mark.hatle@windriver.com>

The layer index module is expected to be used by various parts of the system
in order to access a layerindex-web (such as layers.openembedded.org) and
perform basic processing on the information, such as dependency scanning.

Along with the layerindex implementation are associated tests.  The tests
properly honor BB_SKIP_NETTESTS='yes' to prevent test failures.

Tests Implemented:
   - Branch, LayerItem, LayerBranch, LayerDependency, Recipe, Machine and
      Distro objects
   - LayerIndex setup using the layers.openembedded.org restapi
   - LayerIndex storing and retrieving from a file
   - LayerIndex verify dependency resolution ordering
   - LayerIndex setup using simulated cooker data

Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
---
 bin/bitbake-selftest                               |    6 +-
 lib/layerindexlib/README                           |   28 +
 lib/layerindexlib/__init__.py                      | 1364 ++++++++++++++++++++
 lib/layerindexlib/cooker.py                        |  341 +++++
 lib/layerindexlib/plugin.py                        |   60 +
 lib/layerindexlib/restapi.py                       |  398 ++++++
 lib/layerindexlib/tests/__init__.py                |    0
 lib/layerindexlib/tests/common.py                  |   43 +
 lib/layerindexlib/tests/cooker.py                  |  123 ++
 lib/layerindexlib/tests/layerindexobj.py           |  226 ++++
 lib/layerindexlib/tests/restapi.py                 |  174 +++
 lib/layerindexlib/tests/testdata/README            |   11 +
 .../tests/testdata/build/conf/bblayers.conf        |   15 +
 .../tests/testdata/layer1/conf/layer.conf          |   17 +
 .../tests/testdata/layer2/conf/layer.conf          |   20 +
 .../tests/testdata/layer3/conf/layer.conf          |   19 +
 .../tests/testdata/layer4/conf/layer.conf          |   22 +
 17 files changed, 2866 insertions(+), 1 deletion(-)
 create mode 100644 lib/layerindexlib/README
 create mode 100644 lib/layerindexlib/__init__.py
 create mode 100644 lib/layerindexlib/cooker.py
 create mode 100644 lib/layerindexlib/plugin.py
 create mode 100644 lib/layerindexlib/restapi.py
 create mode 100644 lib/layerindexlib/tests/__init__.py
 create mode 100644 lib/layerindexlib/tests/common.py
 create mode 100644 lib/layerindexlib/tests/cooker.py
 create mode 100644 lib/layerindexlib/tests/layerindexobj.py
 create mode 100644 lib/layerindexlib/tests/restapi.py
 create mode 100644 lib/layerindexlib/tests/testdata/README
 create mode 100644 lib/layerindexlib/tests/testdata/build/conf/bblayers.conf
 create mode 100644 lib/layerindexlib/tests/testdata/layer1/conf/layer.conf
 create mode 100644 lib/layerindexlib/tests/testdata/layer2/conf/layer.conf
 create mode 100644 lib/layerindexlib/tests/testdata/layer3/conf/layer.conf
 create mode 100644 lib/layerindexlib/tests/testdata/layer4/conf/layer.conf

diff --git a/bin/bitbake-selftest b/bin/bitbake-selftest
index afe1603..7564de3 100755
--- a/bin/bitbake-selftest
+++ b/bin/bitbake-selftest
@@ -22,6 +22,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib
 import unittest
 try:
     import bb
+    import layerindexlib
 except RuntimeError as exc:
     sys.exit(str(exc))
 
@@ -31,7 +32,10 @@ tests = ["bb.tests.codeparser",
          "bb.tests.event",
          "bb.tests.fetch",
          "bb.tests.parse",
-         "bb.tests.utils"]
+         "bb.tests.utils",
+         "layerindexlib.tests.layerindexobj",
+         "layerindexlib.tests.restapi",
+         "layerindexlib.tests.cooker"]
 
 for t in tests:
     t = '.'.join(t.split('.')[:3])
diff --git a/lib/layerindexlib/README b/lib/layerindexlib/README
new file mode 100644
index 0000000..5d927af
--- /dev/null
+++ b/lib/layerindexlib/README
@@ -0,0 +1,28 @@
+The layerindexlib module is designed to permit programs to work directly
+with layer index information.  (See layers.openembedded.org...)
+
+The layerindexlib module includes a plugin interface that is used to extend
+the basic functionality.  There are two primary plugins available: restapi
+and cooker.
+
+The restapi plugin works with a web based REST Api compatible with the
+layerindex-web project, as well as the ability to store and retried a
+the information for one or more files on the disk.
+
+The cooker plugin works by reading the information from the current build
+project and processing it as if it were a layer index.
+
+
+TODO:
+
+__init__.py:
+Implement local on-disk caching (using the rest api store/load)
+Implement layer index style query operations on a combined index
+
+common.py:
+Stop network access if BB_NO_NETWORK or allowed hosts is restricted
+
+cooker.py:
+Cooker - Implement recipe parsing
+
+
diff --git a/lib/layerindexlib/__init__.py b/lib/layerindexlib/__init__.py
new file mode 100644
index 0000000..74f3e2e
--- /dev/null
+++ b/lib/layerindexlib/__init__.py
@@ -0,0 +1,1364 @@
+# Copyright (C) 2016-2018 Wind River Systems, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import datetime
+
+import logging
+import imp
+
+from collections import OrderedDict
+from layerindexlib.plugin import LayerIndexPluginUrlError
+
+logger = logging.getLogger('BitBake.layerindexlib')
+
+# Exceptions
+
+class LayerIndexException(Exception):
+    '''LayerIndex Generic Exception'''
+    def __init__(self, message):
+         self.msg = message
+         Exception.__init__(self, message)
+
+    def __str__(self):
+         return self.msg
+
+class LayerIndexUrlError(LayerIndexException):
+    '''Exception raised when unable to access a URL for some reason'''
+    def __init__(self, url, message=""):
+        if message:
+            msg = "Unable to access layerindex url %s: %s" % (url, message)
+        else:
+            msg = "Unable to access layerindex url %s" % url
+        self.url = url
+        LayerIndexException.__init__(self, msg)
+
+class LayerIndexFetchError(LayerIndexException):
+    '''General layerindex fetcher exception when something fails'''
+    def __init__(self, url, message=""):
+        if message:
+            msg = "Unable to fetch layerindex url %s: %s" % (url, message)
+        else:
+            msg = "Unable to fetch layerindex url %s" % url
+        self.url = url
+        LayerIndexException.__init__(self, msg)
+
+
+# Interface to the overall layerindex system
+# the layer may contain one or more individual indexes
+class LayerIndex():
+    def __init__(self, d):
+        if not d:
+            raise LayerIndexException("Must be initialized with bb.data.")
+
+        self.data = d
+
+        # List of LayerIndexObj
+        self.indexes = []
+
+        self.plugins = []
+
+        import bb.utils
+        bb.utils.load_plugins(logger, self.plugins, os.path.dirname(__file__))
+        for plugin in self.plugins:
+            if hasattr(plugin, 'init'):
+                plugin.init(self)
+
+    def __add__(self, other):
+        newIndex = LayerIndex(self.data)
+
+        if self.__class__ != newIndex.__class__ or \
+           other.__class__ != newIndex.__class__:
+            raise TypeException("Can not add different types.")
+
+        for indexEnt in self.indexes:
+            newIndex.indexes.append(indexEnt)
+
+        for indexEnt in other.indexes:
+            newIndex.indexes.append(indexEnt)
+
+        return newIndex
+
+    def _parse_params(self, params):
+        '''Take a parameter list, return a dictionary of parameters.
+
+           Expected to be called from the data of urllib.parse.urlparse(url).params
+
+           If there are two conflicting parameters, last in wins...
+        '''
+
+        param_dict = {}
+        for param in params.split(';'):
+           if not param:
+               continue
+           item = param.split('=', 1)
+           logger.debug(1, item)
+           param_dict[item[0]] = item[1]
+
+        return param_dict
+
+    def _fetch_url(self, url, username=None, password=None, debuglevel=0):
+        '''Fetch data from a specific URL.
+
+           Fetch something from a specific URL.  This is specifically designed to
+           fetch data from a layerindex-web instance, but may be useful for other
+           raw fetch actions.
+
+           It is not designed to be used to fetch recipe sources or similar.  the
+           regular fetcher class should used for that.
+
+           It is the responsibility of the caller to check BB_NO_NETWORK and related
+           BB_ALLOWED_NETWORKS.
+        '''
+
+        if not url:
+            raise LayerIndexUrlError(url, "empty url")
+
+        import urllib
+        from urllib.request import urlopen, Request
+        from urllib.parse import urlparse
+
+        up = urlparse(url)
+
+        if username:
+            logger.debug(1, "Configuring authentication for %s..." % url)
+            password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
+            password_mgr.add_password(None, "%s://%s" % (up.scheme, up.netloc), username, password)
+            handler = urllib.request.HTTPBasicAuthHandler(password_mgr)
+            opener = urllib.request.build_opener(handler, urllib.request.HTTPSHandler(debuglevel=debuglevel))
+        else:
+            opener = urllib.request.build_opener(urllib.request.HTTPSHandler(debuglevel=debuglevel))
+
+        urllib.request.install_opener(opener)
+
+        logger.debug(1, "Fetching %s (%s)..." % (url, ["without authentication", "with authentication"][bool(username)]))
+
+        try:
+            res = urlopen(Request(url, headers={'User-Agent': 'Mozilla/5.0 (bitbake/lib/layerindex)'}, unverifiable=True))
+        except urllib.error.HTTPError as e:
+            logger.debug(1, "HTTP Error: %s: %s" % (e.code, e.reason))
+            logger.debug(1, " Requested: %s" % (url))
+            logger.debug(1, " Actual:    %s" % (e.geturl()))
+
+            if e.code == 404:
+                logger.debug(1, "Request not found.")
+                raise LayerIndexFetchError(url, e)
+            else:
+                logger.debug(1, "Headers:\n%s" % (e.headers))
+                raise LayerIndexFetchError(url, e)
+        except OSError as e:
+            error = 0
+            reason = ""
+
+            # Process base OSError first...
+            if hasattr(e, 'errno'):
+                error = e.errno
+                reason = e.strerror
+
+            # Process gaierror (socket error) subclass if available.
+            if hasattr(e, 'reason') and hasattr(e.reason, 'errno') and hasattr(e.reason, 'strerror'):
+                error = e.reason.errno
+                reason = e.reason.strerror
+                if error == -2:
+                    raise LayerIndexFetchError(url, "%s: %s" % (e, reason))
+
+            if error and error != 0:
+                raise LayerIndexFetchError(url, "Unexpected exception: [Error %s] %s" % (error, reason))
+            else:
+                raise LayerIndexFetchError(url, "Unable to fetch OSError exception: %s" % e)
+
+        finally:
+            logger.debug(1, "...fetching %s (%s), done." % (url, ["without authentication", "with authentication"][bool(username)]))
+
+        return res
+
+
+    def load_layerindex(self, indexURI, load=['layerDependencies', 'recipes', 'machines', 'distros'], reload=False):
+        '''Load the layerindex.
+
+           indexURI - An index to load.  (Use multiple calls to load multiple indexes)
+           
+           reload - If reload is True, then any previously loaded indexes will be forgotten.
+           
+           load - List of elements to load.  Default loads all items.
+                  Note: plugs may ignore this.
+
+The format of the indexURI:
+
+  <url>;branch=<branch>;cache=<cache>;desc=<description>
+
+  Note: the 'branch' parameter if set can select multiple branches by using
+  comma, such as 'branch=master,morty,pyro'.  However, many operations only look
+  at the -first- branch specified!
+
+  The cache value may be undefined, in this case a network failure will
+  result in an error, otherwise the system will look for a file of the cache
+  name and load that instead.
+
+  For example:
+
+  http://layers.openembedded.org/layerindex/api/;branch=master;desc=OpenEmbedded%20Layer%20Index
+  cooker://
+'''
+        if reload:
+            self.indexes = []
+
+        logger.debug(1, 'Loading: %s' % indexURI)
+
+        if not self.plugins:
+            raise LayerIndexException("No LayerIndex Plugins available")
+
+        for plugin in self.plugins:
+            # Check if the plugin was initialized
+            logger.debug(1, 'Trying %s' % plugin.__class__)
+            if not hasattr(plugin, 'type') or not plugin.type:
+                continue
+            try:
+                # TODO: Implement 'cache', for when the network is not available
+                indexEnt = plugin.load_index(indexURI, load)
+                break
+            except LayerIndexPluginUrlError as e:
+                logger.debug(1, "%s doesn't support %s" % (plugin.type, e.url))
+            except NotImplementedError:
+                pass
+        else:
+            logger.debug(1, "No plugins support %s" % indexURI)
+            raise LayerIndexException("No plugins support %s" % indexURI)
+
+        # Mark CONFIG data as something we've added...
+        indexEnt.config['local'] = []
+        indexEnt.config['local'].append('config')
+
+        # No longer permit changes..
+        indexEnt.lockData()
+
+        self.indexes.append(indexEnt)
+
+    def store_layerindex(self, indexURI, index=None):
+        '''Store one layerindex
+
+Typically this will be used to create a local cache file of a remote index.
+
+  file://<path>;branch=<branch>
+
+We can write out in either the restapi or django formats.  The split option
+will write out the individual elements split by layer and related components.
+'''
+        if not index:
+            logger.warning('No index to write, nothing to do.')
+            return
+
+        if not self.plugins:
+            raise LayerIndexException("No LayerIndex Plugins available")
+
+        for plugin in self.plugins:
+            # Check if the plugin was initialized
+            logger.debug(1, 'Trying %s' % plugin.__class__)
+            if not hasattr(plugin, 'type') or not plugin.type:
+                continue
+            try:
+                plugin.store_index(indexURI, index)
+                break
+            except LayerIndexPluginUrlError as e:
+                logger.debug(1, "%s doesn't support %s" % (plugin.type, e.url))
+            except NotImplementedError:
+                logger.debug(1, "Store not implemented in %s" % plugin.type)
+                pass
+        else:
+            logger.debug(1, "No plugins support %s" % url)
+            raise LayerIndexException("No plugins support %s" % url)
+
+
+    def is_empty(self):
+        '''Return True or False if the index has any usable data.
+
+We check the indexes entries to see if they have a branch set, as well as
+layerBranches set.  If not, they are effectively blank.'''
+
+        found = False
+        for index in self.indexes:
+            if index.__bool__():
+                found = True
+                break
+        return not found
+
+
+    def find_vcs_url(self, vcs_url, branch=None):
+        '''Return the first layerBranch with the given vcs_url
+
+           If a branch has not been specified, we will iterate over the branches in
+           the default configuration until the first vcs_url/branch match.'''
+
+        for index in self.indexes:
+            logger.debug(1, ' searching %s' % index.config['DESCRIPTION'])
+            layerBranch = index.find_vcs_url(vcs_url, [branch])
+            if layerBranch:
+                return layerBranch
+        return None
+
+    def find_collection(self, collection, version=None, branch=None):
+        '''Return the first layerBranch with the given collection name
+
+           If a branch has not been specified, we will iterate over the branches in
+           the default configuration until the first collection/branch match.'''
+
+        logger.debug(1, 'find_collection: %s (%s) %s' % (collection, version, branch))
+
+        if branch:
+            branches = [branch]
+        else:
+            branches = None
+
+        for index in self.indexes:
+            logger.debug(1, ' searching %s' % index.config['DESCRIPTION'])
+            layerBranch = index.find_collection(collection, version, branches)
+            if layerBranch:
+                return layerBranch
+        else:
+            logger.debug(1, 'Collection %s (%s) not found for branch (%s)' % (collection, version, branch))
+        return None
+
+    def find_layerbranch(self, name, branch=None):
+        '''Return the layerBranch item for a given name and branch
+
+           If a branch has not been specified, we will iterate over the branches in
+           the default configuration until the first name/branch match.'''
+
+        if branch:
+            branches = [branch]
+        else:
+            branches = None
+
+        for index in self.indexes:
+            layerBranch = index.find_layerbranch(name, branches)
+            if layerBranch:
+                return layerBranch
+        return None
+
+    def find_dependencies(self, names=None, layerbranches=None, ignores=None):
+        '''Return a tuple of all dependencies and valid items for the list of (layer) names
+
+        The dependency scanning happens depth-first.  The returned
+        dependencies should be in the best order to define bblayers.
+
+          names - list of layer names (searching layerItems)
+          branches - when specified (with names) only this list of branches are evaluated
+
+          layerbranches - list of layerbranches to resolve dependencies
+
+          ignores - list of layer names to ignore
+
+        return: (dependencies, invalid)
+
+          dependencies[LayerItem.name] = [ LayerBranch, LayerDependency1, LayerDependency2, ... ]
+          invalid = [ LayerItem.name1, LayerItem.name2, ... ]
+        '''
+
+        invalid = []
+
+        # Convert name/branch to layerbranches
+        if layerbranches is None:
+            layerbranches = []
+
+        for name in names:
+            if ignores and name in ignores:
+                continue
+
+            for index in self.indexes:
+                layerbranch = index.find_layerbranch(name)
+                if not layerbranch:
+                    # Not in this index, hopefully it's in another...
+                    continue
+                layerbranches.append(layerbranch)
+                break
+            else:
+                invalid.append(name)
+
+
+        def _resolve_dependencies(layerbranches, ignores, dependencies, invalid):
+            for layerbranch in layerbranches:
+                if ignores and layerbranch.layer.name in ignores:
+                    continue
+
+                # Get a list of dependencies and then recursively process them
+                for layerdependency in layerbranch.index.layerDependencies_layerBranchId[layerbranch.id]:
+                    deplayerbranch = layerdependency.dependency_layerBranch
+
+                    if ignores and deplayerbranch.layer.name in ignores:
+                        continue
+
+                    # This little block is why we can't re-use the LayerIndexObj version,
+                    # we must be able to satisfy each dependencies across layer indexes and
+                    # use the layer index order for priority.  (r stands for replacement below)
+
+                    # If this is the primary index, we can fast path and skip this
+                    if deplayerbranch.index != self.indexes[0]:
+                        # Is there an entry in a prior index for this collection/version?
+                        rdeplayerbranch = self.find_collection(
+                                              collection=deplayerbranch.collection,
+                                              version=deplayerbranch.version
+                                          )
+                        if rdeplayerbranch != deplayerbranch:
+                                logger.debug(1, 'Replaced %s:%s:%s with %s:%s:%s' % \
+                                      (deplayerbranch.index.config['DESCRIPTION'],
+                                       deplayerbranch.branch.name,
+                                       deplayerbranch.layer.name,
+                                       rdeplayerbranch.index.config['DESCRIPTION'],
+                                       rdeplayerbranch.branch.name,
+                                       rdeplayerbranch.layer.name))
+                                deplayerbranch = rdeplayerbranch
+
+                    # New dependency, we need to resolve it now... depth-first
+                    if deplayerbranch.layer.name not in dependencies:
+                        (dependencies, invalid) = _resolve_dependencies([deplayerbranch], ignores, dependencies, invalid)
+
+                    if deplayerbranch.layer.name not in dependencies:
+                        dependencies[deplayerbranch.layer.name] = [deplayerbranch, layerdependency]
+                    else:
+                        if layerdependency not in dependencies[deplayerbranch.layer.name]:
+                            dependencies[deplayerbranch.layer.name].append(layerdependency)
+
+            return (dependencies, invalid)
+
+        # OK, resolve this one...
+        dependencies = OrderedDict()
+        (dependencies, invalid) = _resolve_dependencies(layerbranches, ignores, dependencies, invalid)
+
+        for layerbranch in layerbranches:
+            if layerbranch.layer.name not in dependencies:
+                dependencies[layerbranch.layer.name] = [layerbranch]
+
+        return (dependencies, invalid)
+
+
+    def list_obj(self, object):
+        '''Print via the plain logger object information
+
+This function is used to implement debugging and provide the user info.
+'''
+        for lix in self.indexes:
+            if object not in lix:
+                continue
+
+            logger.plain ('')
+            logger.plain ('Index: %s' % lix.config['DESCRIPTION'])
+
+            output = []
+
+            if object == 'branches':
+                logger.plain ('%s %s %s' % ('{:26}'.format('branch'), '{:34}'.format('description'), '{:22}'.format('bitbake branch')))
+                logger.plain ('{:-^80}'.format(""))
+                for branchid in lix.branches:
+                    output.append('%s %s %s' % (
+                                  '{:26}'.format(lix.branches[branchid].name),
+                                  '{:34}'.format(lix.branches[branchid].short_description),
+                                  '{:22}'.format(lix.branches[branchid].bitbake_branch)
+                                 ))
+                for line in sorted(output):
+                    logger.plain (line)
+
+                continue
+
+            if object == 'layerItems':
+                logger.plain ('%s %s' % ('{:26}'.format('layer'), '{:34}'.format('description')))
+                logger.plain ('{:-^80}'.format(""))
+                for layerid in lix.layerItems:
+                    output.append('%s %s' % (
+                                  '{:26}'.format(lix.layerItems[layerid].name),
+                                  '{:34}'.format(lix.layerItems[layerid].summary)
+                                 ))
+                for line in sorted(output):
+                    logger.plain (line)
+
+                continue
+
+            if object == 'layerBranches':
+                logger.plain ('%s %s %s' % ('{:26}'.format('layer'), '{:34}'.format('description'), '{:19}'.format('collection:version')))
+                logger.plain ('{:-^80}'.format(""))
+                for layerbranchid in lix.layerBranches:
+                    output.append('%s %s %s' % (
+                                  '{:26}'.format(lix.layerBranches[layerbranchid].layer.name),
+                                  '{:34}'.format(lix.layerBranches[layerbranchid].layer.summary),
+                                  '{:19}'.format("%s:%s" %
+                                                          (lix.layerBranches[layerbranchid].collection,
+                                                           lix.layerBranches[layerbranchid].version)
+                                                )
+                                 ))
+                for line in sorted(output):
+                    logger.plain (line)
+
+                continue
+
+            if object == 'layerDependencies':
+                logger.plain ('%s %s %s %s' % ('{:19}'.format('branch'), '{:26}'.format('layer'), '{:11}'.format('dependency'), '{:26}'.format('layer')))
+                logger.plain ('{:-^80}'.format(""))
+                for layerDependency in lix.layerDependencies:
+                    if not lix.layerDependencies[layerDependency].dependency_layerBranch:
+                        continue
+
+                    output.append('%s %s %s %s' % (
+                                  '{:19}'.format(lix.layerDependencies[layerDependency].layerbranch.branch.name),
+                                  '{:26}'.format(lix.layerDependencies[layerDependency].layerbranch.layer.name),
+                                  '{:11}'.format('requires' if lix.layerDependencies[layerDependency].required else 'recommends'),
+                                  '{:26}'.format(lix.layerDependencies[layerDependency].dependency_layerBranch.layer.name)
+                                 ))
+                for line in sorted(output):
+                    logger.plain (line)
+
+                continue
+
+            if object == 'recipes':
+                logger.plain ('%s %s %s' % ('{:20}'.format('recipe'), '{:10}'.format('version'), 'layer'))
+                logger.plain ('{:-^80}'.format(""))
+                output = []
+                for recipe in lix.recipes:
+                    output.append('%s %s %s' % (
+                                  '{:30}'.format(lix.recipes[recipe].pn),
+                                  '{:30}'.format(lix.recipes[recipe].pv),
+                                  lix.recipes[recipe].layer.name
+                                 ))
+                for line in sorted(output):
+                    logger.plain (line)
+
+                continue
+
+            if object == 'machines':
+                logger.plain ('%s %s %s' % ('{:24}'.format('machine'), '{:34}'.format('description'), '{:19}'.format('layer')))
+                logger.plain ('{:-^80}'.format(""))
+                for machine in lix.machines:
+                    output.append('%s %s %s' % (
+                                  '{:24}'.format(lix.machines[machine].name),
+                                  '{:34}'.format(lix.machines[machine].description)[:34],
+                                  '{:19}'.format(lix.machines[machine].layerbranch.layer.name)
+                                 ))
+                for line in sorted(output):
+                    logger.plain (line)
+
+                continue
+
+            if object == 'distros':
+                logger.plain ('%s %s %s' % ('{:24}'.format('distro'), '{:34}'.format('description'), '{:19}'.format('layer')))
+                logger.plain ('{:-^80}'.format(""))
+                for distro in lix.distros:
+                    output.append('%s %s %s' % (
+                                  '{:24}'.format(lix.distros[distro].name),
+                                  '{:34}'.format(lix.distros[distro].description)[:34],
+                                  '{:19}'.format(lix.distros[distro].layerbranch.layer.name)
+                                 ))
+                for line in sorted(output):
+                    logger.plain (line)
+
+                continue
+
+        logger.plain ('')
+
+
+# This class holds a single layer index instance
+# The LayerIndexObj is made up of dictionary of elements, such as:
+#   index['config'] - configuration data for this index
+#   index['branches'] - dictionary of Branch objects, by id number
+#   index['layerItems'] - dictionary of layerItem objects, by id number
+#   ...etc...  (See: http://layers.openembedded.org/layerindex/api/)
+#
+# The class needs to manage the 'index' entries and allow easily adding
+# of new items, as well as simply loading of the items.
+class LayerIndexObj():
+    def __init__(self):
+        super().__setattr__('_index', {})
+        super().__setattr__('_lock', False)
+
+    def __bool__(self):
+        '''False if the index is effectively empty
+
+           We check the index to see if it has a branch set, as well as
+           layerbranches set.  If not, it is effectively blank.'''
+
+        if not bool(self._index):
+            return False
+
+        try:
+            if self.branches and self.layerBranches:
+                return True
+        except AttributeError:
+            pass
+
+        return False
+
+    def __getattr__(self, name):
+        if name.startswith('_'):
+            return super().__getattribute__(name)
+
+        if name not in self._index:
+            raise AttributeError('%s not in index datastore' % name)
+
+        return self._index[name]
+
+    def __setattr__(self, name, value):
+        if self.isLocked():
+            raise TypeError("Can not set attribute '%s': index is locked" % name)
+
+        if name.startswith('_'):
+            super().__setattr__(name, value)
+            return
+
+        self._index[name] = value
+
+    def __delattr__(self, name):
+        if self.isLocked():
+            raise TypeError("Can not delete attribute '%s': index is locked" % name)
+
+        if name.startswith('_'):
+            super().__delattr__(name)
+
+        self._index.pop(name)
+
+    def lockData(self):
+        '''Lock data object (make it readonly)'''
+        super().__setattr__("_lock", True)
+
+    def unlockData(self):
+        '''unlock data object (make it readonly)'''
+        super().__setattr__("_lock", False)
+
+        # When the data is unlocked, we have to clear the caches, as
+        # modification is allowed!
+        del(self._layerBranches_layerId_branchId)
+        del(self._layerDependencies_layerBranchId)
+        del(self._layerBranches_vcsUrl)
+
+    def isLocked(self):
+        '''Is this object locked (readonly)?'''
+        return self._lock
+
+    def add_element(self, indexname, objs):
+        '''Add a layer index object to index.<indexname>'''
+        if indexname not in self._index:
+            self._index[indexname] = {}
+
+        for obj in objs:
+            if obj.id in self._index[indexname]:
+                if self._index[indexname][obj.id] == obj:
+                    continue
+                raise LayerIndexError('Conflict adding object %s(%s) to index' % (indexname, obj.id))
+            self._index[indexname][obj.id] = obj
+
+    def add_raw_element(self, indexname, objtype, rawobjs):
+        '''Convert a raw layer index data item to a layer index item object and add to the index'''
+        objs = []
+        for entry in rawobjs:
+            objs.append(objtype(self, entry))
+        self.add_element(indexname, objs)
+
+    # Quick lookup table for searching layerId and branchID combos
+    @property
+    def layerBranches_layerId_branchId(self):
+        def createCache(self):
+            cache = {}
+            for layerbranchid in self.layerBranches:
+                layerbranch = self.layerBranches[layerbranchid]
+                cache["%s:%s" % (layerbranch.layer_id, layerbranch.branch_id)] = layerbranch
+            return cache
+
+        if self.isLocked():
+            cache = getattr(self, '_layerBranches_layerId_branchId', None)
+        else:
+            cache = None
+
+        if not cache:
+            cache = createCache(self)
+
+        if self.isLocked():
+            super().__setattr__('_layerBranches_layerId_branchId', cache)
+
+        return cache
+
+    # Quick lookup table for finding all dependencies of a layerBranch
+    @property
+    def layerDependencies_layerBranchId(self):
+        def createCache(self):
+            cache = {}
+            # This ensures empty lists for all branchids
+            for layerbranchid in self.layerBranches:
+                cache[layerbranchid] = []
+
+            for layerdependencyid in self.layerDependencies:
+                layerdependency = self.layerDependencies[layerdependencyid]
+                cache[layerdependency.layerbranch_id].append(layerdependency)
+            return cache
+
+        if self.isLocked():
+            cache = getattr(self, '_layerDependencies_layerBranchId', None)
+        else:
+            cache = None
+
+        if not cache:
+            cache = createCache(self)
+
+        if self.isLocked():
+            super().__setattr__('_layerDependencies_layerBranchId', cache)
+
+        return cache
+
+    # Quick lookup table for finding all instances of a vcs_url
+    @property
+    def layerBranches_vcsUrl(self):
+        def createCache(self):
+            cache = {}
+            for layerbranchid in self.layerBranches:
+                layerbranch = self.layerBranches[layerbranchid]
+                if layerbranch.layer.vcs_url not in cache:
+                   cache[layerbranch.layer.vcs_url] = [layerbranch]
+                else:
+                   cache[layerbranch.layer.vcs_url].append(layerbranch)
+            return cache
+
+        if self.isLocked():
+            cache = getattr(self, '_layerBranches_vcsUrl', None)
+        else:
+            cache = None
+
+        if not cache:
+            cache = createCache(self)
+
+        if self.isLocked():
+            super().__setattr__('_layerBranches_vcsUrl', cache)
+
+        return cache
+
+
+    def find_vcs_url(self, vcs_url, branches=None):
+        ''''Return the first layerBranch with the given vcs_url
+
+            If a list of branches has not been specified, we will iterate on
+            all branches until the first vcs_url is found.'''
+
+        if not self.__bool__():
+            return None
+
+        for layerbranch in self.layerBranches_vcsUrl:
+            if branches and layerbranch.branch.name not in branches:
+                continue
+
+            return layerbranch
+
+        return None
+
+
+    def find_collection(self, collection, version=None, branches=None):
+        '''Return the first layerBranch with the given collection name
+
+           If a list of branches has not been specified, we will iterate on
+           all branches until the first collection is found.'''
+
+        if not self.__bool__():
+            return None
+
+        for layerbranchid in self.layerBranches:
+            layerbranch = self.layerBranches[layerbranchid]
+            if branches and layerbranch.branch.name not in branches:
+                continue
+
+            if layerbranch.collection == collection and \
+                (version is None or version == layerbranch.version):
+                return layerbranch
+
+        return None
+
+
+    def find_layerbranch(self, name, branches=None):
+        '''Return the first layerbranch whose layer name matches
+
+           If a list of branches has not been specified, we will iterate on
+           all branches until the first layer with that name is found.'''
+
+        if not self.__bool__():
+            return None
+
+        for layerbranchid in self.layerBranches:
+            layerbranch = self.layerBranches[layerbranchid]
+            if branches and layerbranch.branch.name not in branches:
+                continue
+
+            if layerbranch.layer.name == name:
+                return layerbranch
+
+        return None
+
+    def find_dependencies(self, names=None, branches=None, layerBranches=None, ignores=None):
+        '''Return a tuple of all dependencies and valid items for the list of (layer) names
+
+        The dependency scanning happens depth-first.  The returned
+        dependencies should be in the best order to define bblayers.
+
+          names - list of layer names (searching layerItems)
+          branches - when specified (with names) only this list of branches are evaluated
+
+          layerBranches - list of layerBranches to resolve dependencies
+
+          ignores - list of layer names to ignore
+
+        return: (dependencies, invalid)
+
+          dependencies[LayerItem.name] = [ LayerBranch, LayerDependency1, LayerDependency2, ... ]
+          invalid = [ LayerItem.name1, LayerItem.name2, ... ]'''
+
+        invalid = []
+
+        # Convert name/branch to layerBranches
+        if layerbranches is None:
+            layerbranches = []
+
+        for name in names:
+            if ignores and name in ignores:
+                continue
+
+            layerbranch = self.find_layerbranch(name, branches)
+            if not layerbranch:
+                invalid.append(name)
+            else:
+                layerbranches.append(layerbranch)
+
+        for layerbranch in layerbranches:
+            if layerbranch.index != self:
+                raise LayerIndexException("Can not resolve dependencies across indexes with this class function!")
+
+        def _resolve_dependencies(layerbranches, ignores, dependencies, invalid):
+            for layerbranch in layerbranches:
+                if ignores and layerBranch.layer.name in ignores:
+                    continue
+
+                for layerdependency in layerbranch.index.layerDependencies_layerBranchId[layerBranch.id]:
+                    deplayerbranch = layerDependency.dependency_layerBranch
+
+                    if ignores and deplayerbranch.layer.name in ignores:
+                        continue
+
+                    # New dependency, we need to resolve it now... depth-first
+                    if deplayerbranch.layer.name not in dependencies:
+                        (dependencies, invalid) = _resolve_dependencies([deplayerbranch], ignores, dependencies, invalid)
+
+                    if deplayerbranch.layer.name not in dependencies:
+                        dependencies[deplayerbranch.layer.name] = [deplayerbranch, layerdependency]
+                    else:
+                        if layerdependency not in dependencies[deplayerbranch.layer.name]:
+                            dependencies[deplayerbranch.layer.name].append(layerdependency)
+
+                return (dependencies, invalid)
+
+        # OK, resolve this one...
+        dependencies = OrderedDict()
+        (dependencies, invalid) = _resolve_dependencies(layerbranches, ignores, dependencies, invalid)
+
+        # Is this item already in the list, if not add it
+        for layerbranch in layerbranches:
+            if layerbranch.layer.name not in dependencies:
+                dependencies[layerbranch.layer.name] = [layerbranch]
+
+        return (dependencies, invalid)
+
+
+# Define a basic LayerIndexItemObj.  This object forms the basis for all other
+# objects.  The raw Layer Index data is stored in the _data element, but we
+# do not want users to access data directly.  So wrap this and protect it
+# from direct manipulation.
+#
+# It is up to the insantiators of the objects to fill them out, and once done
+# lock the objects to prevent further accidently manipulation.
+#
+# Using the getattr, setattr and properties we can access and manipulate
+# the data within the data element.
+class LayerIndexItemObj():
+    def __init__(self, index, data=None, lock=False):
+        if data is None:
+            data = {}
+
+        if type(data) != type(dict()):
+            raise TypeError('data (%s) is not a dict' % type(data))
+
+        super().__setattr__('_lock',  lock)
+        super().__setattr__('index', index)
+        super().__setattr__('_data',  data)
+
+    def __eq__(self, other):
+        if self.__class__ != other.__class__:
+            return False
+        res=(self._data == other._data)
+        return res
+
+    def __bool__(self):
+        return bool(self._data)
+
+    def __getattr__(self, name):
+        # These are internal to THIS class, and not part of data
+        if name == "index" or name.startswith('_'):
+            return super().__getattribute__(name)
+
+        if name not in self._data:
+            raise AttributeError('%s not in datastore' % name)
+
+        return self._data[name]
+
+    def _setattr(self, name, value, prop=True):
+        '''__setattr__ like function, but with control over property object behavior'''
+        if self.isLocked():
+            raise TypeError("Can not set attribute '%s': Object data is locked" % name)
+
+        if name.startswith('_'):
+            super().__setattr__(name, value)
+            return
+
+        # Since __setattr__ runs before properties, we need to check if
+        # there is a setter property and then execute it
+        # ... or return self._data[name]
+        propertyobj = getattr(self.__class__, name, None)
+        if prop and isinstance(propertyobj, property):
+            if propertyobj.fset:
+                propertyobj.fset(self, value)
+            else:
+                raise AttributeError('Attribute %s is readonly, and may not be set' % name)
+        else:
+            self._data[name] = value
+
+    def __setattr__(self, name, value):
+        self._setattr(name, value, prop=True)
+
+    def _delattr(self, name, prop=True):
+        # Since __delattr__ runs before properties, we need to check if
+        # there is a deleter property and then execute it
+        # ... or we pop it ourselves..
+        propertyobj = getattr(self.__class__, name, None)
+        if prop and isinstance(propertyobj, property):
+            if propertyobj.fdel:
+                propertyobj.fdel(self)
+            else:
+                raise AttributeError('Attribute %s is readonly, and may not be deleted' % name)
+        else:
+            self._data.pop(name)
+
+    def __delattr__(self, name):
+        self._delattr(name, prop=True)
+
+    def lockData(self):
+        '''Lock data object (make it readonly)'''
+        super().__setattr__("_lock", True)
+
+    def unlockData(self):
+        '''unlock data object (make it readonly)'''
+        super().__setattr__("_lock", False)
+
+    def isLocked(self):
+        '''Is this object locked (readonly)?'''
+        return self._lock
+
+# Branch object
+class Branch(LayerIndexItemObj):
+    def define_data(self, id, name, bitbake_branch,
+                 short_description=None, sort_priority=1,
+                 updates_enabled=True, updated=None,
+                 update_environment=None):
+        self.id = id
+        self.name = name
+        self.bitbake_branch = bitbake_branch
+        self.short_description = short_description or name
+        self.sort_priority = sort_priority
+        self.updates_enabled = updates_enabled
+        self.updated = updated or datetime.datetime.today().isoformat()
+        self.update_environment = update_environment
+
+    @property
+    def name(self):
+        return self.__getattr__('name')
+
+    @name.setter
+    def name(self, value):
+        self._data['name'] = value
+
+        if self.bitbake_branch == value:
+            self.bitbake_branch = ""
+
+    @name.deleter
+    def name(self):
+        self._delattr('name', prop=False)
+
+    @property
+    def bitbake_branch(self):
+        try:
+            return self.__getattr__('bitbake_branch')
+        except AttributeError:
+            return self.name
+
+    @bitbake_branch.setter
+    def bitbake_branch(self, value):
+        if self.name == value:
+            self._data['bitbake_branch'] = ""
+        else:
+            self._data['bitbake_branch'] = value
+
+    @bitbake_branch.deleter
+    def bitbake_branch(self):
+        self._delattr('bitbake_branch', prop=False)
+
+
+class LayerItem(LayerIndexItemObj):
+    def define_data(self, id, name, status='P',
+                 layer_type='A', summary=None,
+                 description=None,
+                 vcs_url=None, vcs_web_url=None,
+                 vcs_web_tree_base_url=None,
+                 vcs_web_file_base_url=None,
+                 usage_url=None,
+                 mailing_list_url=None,
+                 index_preference=1,
+                 classic=False,
+                 updated=None):
+        self.id = id
+        self.name = name
+        self.status = status
+        self.layer_type = layer_type
+        self.summary = summary or name
+        self.description = description or summary or name
+        self.vcs_url = vcs_url
+        self.vcs_web_url = vcs_web_url
+        self.vcs_web_tree_base_url = vcs_web_tree_base_url
+        self.vcs_web_file_base_url = vcs_web_file_base_url
+        self.index_preference = index_preference
+        self.classic = classic
+        self.updated = updated or datetime.datetime.today().isoformat()
+
+
+class LayerBranch(LayerIndexItemObj):
+    def define_data(self, id, collection, version, layer, branch,
+                 vcs_subdir="", vcs_last_fetch=None,
+                 vcs_last_rev=None, vcs_last_commit=None,
+                 actual_branch="",
+                 updated=None):
+        self.id = id
+        self.collection = collection
+        self.version = version
+        if type(layer) != type(LayerItem):
+            self.layer_id = layer
+        else:
+            self.layer = layer
+
+        if type(branch) != type(Branch):
+            self.branch_id = branch
+        else:
+            self.branch = branch
+
+        self.vcs_subdir = vcs_subdir
+        self.vcs_last_fetch = vcs_last_fetch
+        self.vcs_last_rev = vcs_last_rev
+        self.vcs_last_commit = vcs_last_commit
+        self.actual_branch = actual_branch
+        self.updated = updated or datetime.datetime.today().isoformat()
+
+    # This is a little odd, the _data attribute is 'layer', but it's really
+    # referring to the layer id.. so lets adjust this to make it useful
+    @property
+    def layer_id(self):
+        return self.__getattr__('layer')
+
+    @layer_id.setter
+    def layer_id(self, value):
+        self._setattr('layer', value, prop=False)
+
+    @layer_id.deleter
+    def layer_id(self):
+        self._delattr('layer', prop=False)
+
+    @property
+    def layer(self):
+        try:
+            return self.index.layerItems[self.layer_id]
+        except KeyError:
+            raise AttributeError('Unable to find layerItems in index to map layer_id %s' % self.layer_id)
+        except IndexError:
+            raise AttributeError('Unable to find layer_id %s in index layerItems' % self.layer_id)
+
+    @layer.setter
+    def layer(self, value):
+        if type(value) != type(LayerItem):
+            raise TypeError('value is not a LayerItem')
+        if self.index != value.index:
+            raise AttributeError('Object and value do not share the same index and thus key set.')
+        self.layer_id = value.id
+
+    @layer.deleter
+    def layer(self):
+        del self.layer_id
+
+    @property
+    def branch_id(self):
+        return self.__getattr__('branch')
+
+    @branch_id.setter
+    def branch_id(self, value):
+        self._setattr('branch', value, prop=False)
+
+    @branch_id.deleter
+    def branch_id(self):
+        self._delattr('branch', prop=False)
+
+    @property
+    def branch(self):
+        try:
+            logger.debug(1, "Get branch object from branches[%s]" % (self.branch_id))
+            return self.index.branches[self.branch_id]
+        except KeyError:
+            raise AttributeError('Unable to find branches in index to map branch_id %s' % self.branch_id)
+        except IndexError:
+            raise AttributeError('Unable to find branch_id %s in index branches' % self.branch_id)
+
+    @branch.setter
+    def branch(self, value):
+        if type(value) != type(LayerItem):
+            raise TypeError('value is not a LayerItem')
+        if self.index != value.index:
+            raise AttributeError('Object and value do not share the same index and thus key set.')
+        self.branch_id = value.id
+
+    @branch.deleter
+    def branch(self):
+        del self.branch_id
+
+    @property
+    def actual_branch(self):
+        if self.__getattr__('actual_branch'):
+            return self.__getattr__('actual_branch')
+        else:
+            return self.branch.name
+
+    @actual_branch.setter
+    def actual_branch(self, value):
+        logger.debug(1, "Set actual_branch to %s .. name is %s" % (value, self.branch.name))
+        if value != self.branch.name:
+            self._setattr('actual_branch', value, prop=False)
+        else:
+            self._setattr('actual_branch', '', prop=False)
+
+    @actual_branch.deleter
+    def actual_branch(self):
+        self._delattr('actual_branch', prop=False)
+
+# Extend LayerIndexItemObj with common LayerBranch manipulations
+# All of the remaining LayerIndex objects refer to layerbranch, and it is
+# up to the user to follow that back through the LayerBranch object into
+# the layer object to get various attributes.  So add an intermediate set
+# of attributes that can easily get us the layerbranch as well as layer.
+
+class LayerIndexItemObj_LayerBranch(LayerIndexItemObj):
+    @property
+    def layerbranch_id(self):
+        return self.__getattr__('layerbranch')
+
+    @layerbranch_id.setter
+    def layerbranch_id(self, value):
+        self._setattr('layerbranch', value, prop=False)
+
+    @layerbranch_id.deleter
+    def layerbranch_id(self):
+        self._delattr('layerbranch', prop=False)
+
+    @property
+    def layerbranch(self):
+        try:
+            return self.index.layerBranches[self.layerbranch_id]
+        except KeyError:
+            raise AttributeError('Unable to find layerBranches in index to map layerbranch_id %s' % self.layerbranch_id)
+        except IndexError:
+            raise AttributeError('Unable to find layerbranch_id %s in index branches' % self.layerbranch_id)
+
+    @layerbranch.setter
+    def layerbranch(self, value):
+        if type(value) != type(LayerBranch):
+            raise TypeError('value (%s) is not a layerBranch' % type(value))
+        if self.index != value.index:
+            raise AttributeError('Object and value do not share the same index and thus key set.')
+        self.layerbranch_id = value.id
+
+    @layerbranch.deleter
+    def layerbranch(self):
+        del self.layerbranch_id
+
+    @property
+    def layer_id(self):
+        return self.layerbranch.layer_id
+
+    # Doesn't make sense to set or delete layer_id
+
+    @property
+    def layer(self):
+        return self.layerbranch.layer
+
+    # Doesn't make sense to set or delete layer
+
+
+class LayerDependency(LayerIndexItemObj_LayerBranch):
+    def define_data(self, id, layerbranch, dependency, required=True):
+        self.id = id
+        if type(layerbranch) != type(LayerBranch):
+            self.layerbranch_id = layerbranch
+        else:
+            self.layerbranch = layerbranch
+        if type(dependency) != type(LayerDependency):
+            self.dependency_id = dependency
+        else:
+            self.dependency = dependency
+        self.required = required
+
+    @property
+    def dependency_id(self):
+        return self.__getattr__('dependency')
+
+    @dependency_id.setter
+    def dependency_id(self, value):
+        self._setattr('dependency', value, prop=False)
+
+    @dependency_id.deleter
+    def dependency_id(self):
+        self._delattr('dependency', prop=False)
+
+    @property
+    def dependency(self):
+        try:
+            return self.index.layerItems[self.dependency_id]
+        except KeyError:
+            raise AttributeError('Unable to find layerItems in index to map layerbranch_id %s' % self.dependency_id)
+        except IndexError:
+            raise AttributeError('Unable to find dependency_id %s in index layerItems' % self.dependency_id)
+
+    @dependency.setter
+    def dependency(self, value):
+        if type(value) != type(LayerDependency):
+            raise TypeError('value (%s) is not a dependency' % type(value))
+        if self.index != value.index:
+            raise AttributeError('Object and value do not share the same index and thus key set.')
+        self.dependency_id = value.id
+
+    @dependency.deleter
+    def dependency(self):
+        self._delattr('dependency', prop=False)
+
+    @property
+    def dependency_layerBranch(self):
+        layerid = self.dependency_id
+        branchid = self.layerbranch.branch_id
+
+        try:
+            return self.index.layerBranches_layerId_branchId["%s:%s" % (layerid, branchid)]
+        except IndexError:
+            # layerBranches_layerId_branchId -- but not layerId:branchId
+            raise AttributeError('Unable to find layerId:branchId %s:%s in index layerBranches_layerId_branchId' % (layerid, branchid))
+        except KeyError:
+            raise AttributeError('Unable to find layerId:branchId %s:%s in layerItems and layerBranches' % (layerid, branchid))
+
+    # dependency_layerBranch doesn't make sense to set or del
+
+
+class Recipe(LayerIndexItemObj_LayerBranch):
+    def define_data(self, id,
+                    filename, filepath, pn, pv, layerbranch,
+                    summary="", description="", section="", license="",
+                    homepage="", bugtracker="", provides="", bbclassextend="",
+                    inherits="", blacklisted="", updated=None):
+        self.id = id
+        self.filename = filename
+        self.filepath = filepath
+        self.pn = pn
+        self.pv = pv
+        self.summary = summary
+        self.description = description
+        self.section = section
+        self.license = license
+        self.homepage = homepage
+        self.bugtracker = bugtracker
+        self.provides = provides
+        self.bbclassextend = bbclassextend
+        self.inherits = inherits
+        self.updated = updated or datetime.datetime.today().isoformat()
+        self.blacklisted = blacklisted
+        if type(layerbranch) != type(LayerBranch):
+            self.layerbranch_id = layerbranch
+        else:
+            self.layerbranch = layerbranch
+
+    @property
+    def fullpath(self):
+        return os.path.join(self.filepath, self.filename)
+
+    # Set would need to understand how to split it
+    # del would we del both parts?
+
+    @property
+    def inherits(self):
+        if 'inherits' not in self._data:
+            # Older indexes may not have this, so emulate it
+            if '-image-' in self.pn:
+                return 'image'
+        return self.__getattr__('inherits')
+
+    @inherits.setter
+    def inherits(self, value):
+        return self._setattr('inherits', value, prop=False)
+
+    @inherits.deleter
+    def inherits(self):
+        return self._delattr('inherits', prop=False)
+
+
+class Machine(LayerIndexItemObj_LayerBranch):
+    def define_data(self, id,
+                    name, description, layerbranch,
+                    updated=None):
+        self.id = id
+        self.name = name
+        self.description = description
+        if type(layerbranch) != type(LayerBranch):
+            self.layerbranch_id = layerbranch
+        else:
+            self.layerbranch = layerbranch
+        self.updated = updated or datetime.datetime.today().isoformat()
+
+class Distro(LayerIndexItemObj_LayerBranch):
+    def define_data(self, id,
+                    name, description, layerbranch,
+                    updated=None):
+        self.id = id
+        self.name = name
+        self.description = description
+        if type(layerbranch) != type(LayerBranch):
+            self.layerbranch_id = layerbranch
+        else:
+            self.layerbranch = layerbranch
+        self.updated = updated or datetime.datetime.today().isoformat()
+
+
+# When performing certain actions, we may need to sort the data.
+# This will allow us to keep it consistent from run to run.
+def sort_entry(item):
+    newitem = item
+    try:
+        if type(newitem) == type(dict()):
+            newitem = OrderedDict(sorted(newitem.items(), key=lambda t: t[0]))
+            for index in newitem:
+                newitem[index] = sort_entry(newitem[index])
+        elif type(newitem) == type(list()):
+            newitem.sort(key=lambda obj: obj['id'])
+            for index, _ in enumerate(newitem):
+                newitem[index] = sort_entry(newitem[index])
+    except:
+        logger.error('Sort failed for item %s' % type(item))
+        pass
+
+    return newitem
diff --git a/lib/layerindexlib/cooker.py b/lib/layerindexlib/cooker.py
new file mode 100644
index 0000000..248a597
--- /dev/null
+++ b/lib/layerindexlib/cooker.py
@@ -0,0 +1,341 @@
+# Copyright (C) 2016-2018 Wind River Systems, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import logging
+import json
+
+from collections import OrderedDict, defaultdict
+
+from urllib.parse import unquote, urlparse
+
+import layerindexlib
+
+import layerindexlib.plugin
+
+logger = logging.getLogger('BitBake.layerindexlib.cooker')
+
+import bb.utils
+
+def plugin_init(plugins):
+    return CookerPlugin()
+
+class CookerPlugin(layerindexlib.plugin.IndexPlugin):
+    def __init__(self):
+        self.type = "cooker"
+
+        self.server_connection = None
+        self.ui_module = None
+        self.server = None
+
+    def _run_command(self, command, path, default=None):
+        try:
+            result, _ = bb.process.run(command, cwd=path)
+            result = result.strip()
+        except bb.process.ExecutionError:
+            result = default
+        return result
+
+    def _handle_git_remote(self, remote):
+        if "://" not in remote:
+            if ':' in remote:
+                # This is assumed to be ssh
+                remote = "ssh://" + remote
+            else:
+                # This is assumed to be a file path
+                remote = "file://" + remote
+        return remote
+
+    def _get_bitbake_info(self):
+        """Return a tuple of bitbake information"""
+
+        # Our path SHOULD be .../bitbake/lib/layerindex/cooker.py
+        bb_path = os.path.dirname(__file__) # .../bitbake/lib/layerindex/cooker.py
+        bb_path = os.path.dirname(bb_path)  # .../bitbake/lib/layerindex
+        bb_path = os.path.dirname(bb_path)  # .../bitbake/lib
+        bb_path = os.path.dirname(bb_path)  # .../bitbake
+        bb_path = self._run_command('git rev-parse --show-toplevel', os.path.dirname(__file__), default=bb_path)
+        bb_branch = self._run_command('git rev-parse --abbrev-ref HEAD', bb_path, default="<unknown>")
+        bb_rev = self._run_command('git rev-parse HEAD', bb_path, default="<unknown>")
+        for remotes in self._run_command('git remote -v', bb_path, default="").split("\n"):
+            remote = remotes.split("\t")[1].split(" ")[0]
+            if "(fetch)" == remotes.split("\t")[1].split(" ")[1]:
+                bb_remote = self._handle_git_remote(remote)
+                break
+        else:
+            bb_remote = self._handle_git_remote(bb_path)
+
+        return (bb_remote, bb_branch, bb_rev, bb_path)
+
+    def _load_bblayers(self, branches=None):
+        """Load the BBLAYERS and related collection information"""
+
+        d = self.layerindex.data
+
+        if not branches:
+            raise LayerIndexFetchError("No branches specified for _load_bblayers!")
+
+        index = layerindexlib.LayerIndexObj()
+
+        branchId = 0
+        index.branches = {}
+
+        layerItemId = 0
+        index.layerItems = {}
+
+        layerBranchId = 0
+        index.layerBranches = {}
+
+        bblayers = d.getVar('BBLAYERS').split()
+
+        if not bblayers:
+            # It's blank!  Nothing to process...
+            return index
+
+        collections = d.getVar('BBFILE_COLLECTIONS')
+        layerconfs = d.varhistory.get_variable_items_files('BBFILE_COLLECTIONS', d)
+        bbfile_collections = {layer: os.path.dirname(os.path.dirname(path)) for layer, path in layerconfs.items()}
+
+        (_, bb_branch, _, _) = self._get_bitbake_info()
+
+        for branch in branches:
+            branchId += 1
+            index.branches[branchId] = layerindexlib.Branch(index, None)
+            index.branches[branchId].define_data(branchId, branch, bb_branch)
+
+        for entry in collections.split():
+            layerpath = entry
+            if entry in bbfile_collections:
+                layerpath = bbfile_collections[entry]
+
+            layername = d.getVar('BBLAYERS_LAYERINDEX_NAME_%s' % entry) or os.path.basename(layerpath)
+            layerversion = d.getVar('LAYERVERSION_%s' % entry) or ""
+            layerurl = self._handle_git_remote(layerpath)
+
+            layersubdir = ""
+            layerrev = "<unknown>"
+            layerbranch = "<unknown>"
+
+            if os.path.isdir(layerpath):
+                layerbasepath = self._run_command('git rev-parse --show-toplevel', layerpath, default=layerpath)
+                if os.path.abspath(layerpath) != os.path.abspath(layerbasepath):
+                    layersubdir = os.path.abspath(layerpath)[len(layerbasepath) + 1:]
+
+                layerbranch = self._run_command('git rev-parse --abbrev-ref HEAD', layerpath, default="<unknown>")
+                layerrev = self._run_command('git rev-parse HEAD', layerpath, default="<unknown>")
+
+                for remotes in self._run_command('git remote -v', layerpath, default="").split("\n"):
+                    remote = remotes.split("\t")[1].split(" ")[0]
+                    if "(fetch)" == remotes.split("\t")[1].split(" ")[1]:
+                        layerurl = self._handle_git_remote(remote)
+                        break
+
+            layerItemId += 1
+            index.layerItems[layerItemId] = layerindexlib.LayerItem(index, None)
+            index.layerItems[layerItemId].define_data(layerItemId, layername, description=layerpath, vcs_url=layerurl)
+
+            for branchId in index.branches:
+                layerBranchId += 1
+                index.layerBranches[layerBranchId] = layerindexlib.LayerBranch(index, None)
+                index.layerBranches[layerBranchId].define_data(layerBranchId, entry, layerversion, layerItemId, branchId,
+                                               vcs_subdir=layersubdir, vcs_last_rev=layerrev, actual_branch=layerbranch)
+
+        return index
+
+
+    def load_index(self, url, load):
+        """
+            Fetches layer information from a build configuration.
+
+            The return value is a dictionary containing API,
+            layer, branch, dependency, recipe, machine, distro, information.
+
+            url type should be 'cooker'.
+            url path is ignored
+        """
+
+        up = urlparse(url)
+
+        if up.scheme != 'cooker':
+            raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
+
+        d = self.layerindex.data
+
+        params = self.layerindex._parse_params(up.params)
+
+        # Only reason to pass a branch is to emulate them...
+        if 'branch' in params:
+            branches = params['branch'].split(',')
+        else:
+            branches = ['HEAD']
+
+        logger.debug(1, "Loading cooker data branches %s" % branches)
+
+        index = self._load_bblayers(branches=branches)
+
+        index.config = {}
+        index.config['TYPE'] = self.type
+        index.config['URL'] = url
+
+        if 'desc' in params:
+            index.config['DESCRIPTION'] = unquote(params['desc'])
+        else:
+            index.config['DESCRIPTION'] = 'local'
+
+        if 'cache' in params:
+            index.config['CACHE'] = params['cache']
+
+        index.config['BRANCH'] = branches
+
+        # ("layerDependencies", layerindexlib.LayerDependency)
+        layerDependencyId = 0
+        if "layerDependencies" in load:
+            index.layerDependencies = {}
+            for layerBranchId in index.layerBranches:
+                branchName = index.layerBranches[layerBranchId].branch.name
+                collection = index.layerBranches[layerBranchId].collection
+
+                def add_dependency(layerDependencyId, index, deps, required):
+                    try:
+                        depDict = bb.utils.explode_dep_versions2(deps)
+                    except bb.utils.VersionStringException as vse:
+                        bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (c, str(vse)))
+
+                    for dep, oplist in list(depDict.items()):
+                        # We need to search ourselves, so use the _ version...
+                        depLayerBranch = index.find_collection(dep, branches=[branchName])
+                        if not depLayerBranch:
+                            # Missing dependency?!
+                            logger.error('Missing dependency %s (%s)' % (dep, branchName))
+                            continue
+
+                        # We assume that the oplist matches...
+                        layerDependencyId += 1
+                        layerDependency = layerindexlib.LayerDependency(index, None)
+                        layerDependency.define_data(id=layerDependencyId,
+                                        required=required, layerbranch=layerBranchId,
+                                        dependency=depLayerBranch.layer_id)
+
+                        logger.debug(1, '%s requires %s' % (layerDependency.layer.name, layerDependency.dependency.name))
+                        index.add_element("layerDependencies", [layerDependency])
+
+                    return layerDependencyId
+
+                deps = d.getVar("LAYERDEPENDS_%s" % collection)
+                if deps:
+                    layerDependencyId = add_dependency(layerDependencyId, index, deps, True)
+
+                deps = d.getVar("LAYERRECOMMENDS_%s" % collection)
+                if deps:
+                    layerDependencyId = add_dependency(layerDependencyId, index, deps, False)
+
+        # Need to load recipes here (requires cooker access)
+        recipeId = 0
+        ## TODO: NOT IMPLEMENTED
+        # The code following this is an example of what needs to be
+        # implemented.  However, it does not work as-is.
+        if False and 'recipes' in load:
+            index.recipes = {}
+
+            ret = self.ui_module.main(self.server_connection.connection, self.server_connection.events, config_params)
+
+            all_versions = self._run_command('allProviders')
+
+            all_versions_list = defaultdict(list, all_versions)
+            for pn in all_versions_list:
+                for ((pe, pv, pr), fpath) in all_versions_list[pn]:
+                    realfn = bb.cache.virtualfn2realfn(fpath)
+
+                    filepath = os.path.dirname(realfn[0])
+                    filename = os.path.basename(realfn[0])
+
+                    # This is all HORRIBLY slow, and likely unnecessary
+                    #dscon = self._run_command('parseRecipeFile', fpath, False, [])
+                    #connector = myDataStoreConnector(self, dscon.dsindex)
+                    #recipe_data = bb.data.init()
+                    #recipe_data.setVar('_remote_data', connector)
+
+                    #summary = recipe_data.getVar('SUMMARY')
+                    #description = recipe_data.getVar('DESCRIPTION')
+                    #section = recipe_data.getVar('SECTION')
+                    #license = recipe_data.getVar('LICENSE')
+                    #homepage = recipe_data.getVar('HOMEPAGE')
+                    #bugtracker = recipe_data.getVar('BUGTRACKER')
+                    #provides = recipe_data.getVar('PROVIDES')
+
+                    layer = bb.utils.get_file_layer(realfn[0], self.config_data)
+
+                    depBranchId = collection_layerbranch[layer]
+
+                    recipeId += 1
+                    recipe = layerindexlib.Recipe(index, None)
+                    recipe.define_data(id=recipeId,
+                                   filename=filename, filepath=filepath,
+                                   pn=pn, pv=pv,
+                                   summary=pn, description=pn, section='?',
+                                   license='?', homepage='?', bugtracker='?',
+                                   provides='?', bbclassextend='?', inherits='?',
+                                   blacklisted='?', layerbranch=depBranchId)
+
+                    index = addElement("recipes", [recipe], index)
+
+        # ("machines", layerindexlib.Machine)
+        machineId = 0
+        if 'machines' in load:
+            index.machines = {}
+
+            for layerBranchId in index.layerBranches:
+                # load_bblayers uses the description to cache the actual path...
+                machine_path = index.layerBranches[layerBranchId].getDescription()
+                machine_path = os.path.join(machine_path, 'conf/machine')
+                if os.path.isdir(machine_path):
+                    for (dirpath, _, filenames) in os.walk(machine_path):
+                        # Ignore subdirs...
+                        if not dirpath.endswith('conf/machine'):
+                            continue
+                        for fname in filenames:
+                            if fname.endswith('.conf'):
+                                machineId += 1
+                                machine = layerindexlib.Machine(index, None)
+                                machine.define_data(id=machineId, name=fname[:-5],
+                                                    description=fname[:-5],
+                                                    layerbranch=collection_layerbranch[entry])
+
+                                index.add_element("machines", [machine])
+
+        # ("distros", layerindexlib.Distro)
+        distroId = 0
+        if 'distros' in load:
+            index.distros = {}
+
+            for layerBranchId in index.layerBranches:
+                # load_bblayers uses the description to cache the actual path...
+                distro_path = index.layerBranches[layerBranchId].getDescription()
+                distro_path = os.path.join(distro_path, 'conf/distro')
+                if os.path.isdir(distro_path):
+                    for (dirpath, _, filenames) in os.walk(distro_path):
+                        # Ignore subdirs...
+                        if not dirpath.endswith('conf/distro'):
+                            continue
+                        for fname in filenames:
+                            if fname.endswith('.conf'):
+                                distroId += 1
+                                distro = layerindexlib.Distro(index, None)
+                                distro.define_data(id=distroId, name=fname[:-5],
+                                                    description=fname[:-5],
+                                                    layerbranch=collection_layerbranch[entry])
+
+                                index.add_element("distros", [distro])
+
+        return index
diff --git a/lib/layerindexlib/plugin.py b/lib/layerindexlib/plugin.py
new file mode 100644
index 0000000..92a2e97
--- /dev/null
+++ b/lib/layerindexlib/plugin.py
@@ -0,0 +1,60 @@
+# Copyright (C) 2016-2018 Wind River Systems, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+# The file contains:
+#   LayerIndex exceptions
+#   Plugin base class
+#   Utility Functions for working on layerindex data
+
+import argparse
+import logging
+import os
+import bb.msg
+
+logger = logging.getLogger('BitBake.layerindexlib.plugin')
+
+class LayerIndexPluginException(Exception):
+    """LayerIndex Generic Exception"""
+    def __init__(self, message):
+         self.msg = message
+         Exception.__init__(self, message)
+
+    def __str__(self):
+         return self.msg
+
+class LayerIndexPluginUrlError(LayerIndexPluginException):
+    """Exception raised when a plugin does not support a given URL type"""
+    def __init__(self, plugin, url):
+        msg = "%s does not support %s:" % (plugin, url)
+        self.plugin = plugin
+        self.url = url
+        LayerIndexPluginException.__init__(self, msg)
+
+class IndexPlugin():
+    def __init__(self):
+        self.type = None
+
+    def init(self, layerindex):
+        self.layerindex = layerindex
+
+    def plugin_type(self):
+        return self.type
+
+    def load_index(self, uri):
+        raise NotImplementedError('load_index is not implemented')
+
+    def store_index(self, uri, index):
+        raise NotImplementedError('store_index is not implemented')
+
diff --git a/lib/layerindexlib/restapi.py b/lib/layerindexlib/restapi.py
new file mode 100644
index 0000000..d08eb20
--- /dev/null
+++ b/lib/layerindexlib/restapi.py
@@ -0,0 +1,398 @@
+# Copyright (C) 2016-2018 Wind River Systems, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import logging
+import json
+from urllib.parse import unquote
+from urllib.parse import urlparse
+
+import layerindexlib
+import layerindexlib.plugin
+
+logger = logging.getLogger('BitBake.layerindexlib.restapi')
+
+def plugin_init(plugins):
+    return RestApiPlugin()
+
+class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
+    def __init__(self):
+        self.type = "restapi"
+
+    def load_index(self, url, load):
+        """
+            Fetches layer information from a local or remote layer index.
+
+            The return value is a LayerIndexObj.
+
+            url is the url to the rest api of the layer index, such as:
+            http://layers.openembedded.org/layerindex/api/
+
+            Or a local file...
+        """
+
+        up = urlparse(url)
+
+        if up.scheme == 'file':
+            return self.load_index_file(up, url, load)
+
+        if up.scheme == 'http' or up.scheme == 'https':
+            return self.load_index_web(up, url, load)
+
+        raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
+
+
+    def load_index_file(self, up, url, load):
+        """
+            Fetches layer information from a local file or directory.
+
+            The return value is a LayerIndexObj.
+
+            ud is the parsed url to the local file or directory.
+        """
+        if not os.path.exists(up.path):
+            raise FileNotFoundError(up.path)
+
+        index = layerindexlib.LayerIndexObj()
+
+        index.config = {}
+        index.config['TYPE'] = self.type
+        index.config['URL'] = url
+
+        params = self.layerindex._parse_params(up.params)
+
+        if 'desc' in params:
+            index.config['DESCRIPTION'] = unquote(params['desc'])
+        else:
+            index.config['DESCRIPTION'] = up.path
+
+        if 'cache' in params:
+            index.config['CACHE'] = params['cache']
+
+        if 'branch' in params:
+            branches = params['branch'].split(',')
+            index.config['BRANCH'] = branches
+        else:
+            branches = ['*']
+
+
+        def load_cache(path, index, branches=[]):
+            logger.debug(1, 'Loading json file %s' % path)
+            with open(path, 'rt', encoding='utf-8') as f:
+                pindex = json.load(f)
+
+            # Filter the branches on loaded files...
+            newpBranch = []
+            for branch in branches:
+                if branch != '*':
+                    if 'branches' in pindex:
+                        for br in pindex['branches']:
+                            if br['name'] == branch:
+                                newpBranch.append(br)
+                else:
+                    if 'branches' in pindex:
+                        for br in pindex['branches']:
+                            newpBranch.append(br)
+
+            if newpBranch:
+                index.add_raw_element('branches', layerindexlib.Branch, newpBranch)
+            else:
+                logger.debug(1, 'No matching branches (%s) in index file(s)' % branches)
+                # No matching branches.. return nothing...
+                return
+
+            for (lName, lType) in [("layerItems", layerindexlib.LayerItem),
+                                   ("layerBranches", layerindexlib.LayerBranch),
+                                   ("layerDependencies", layerindexlib.LayerDependency),
+                                   ("recipes", layerindexlib.Recipe),
+                                   ("machines", layerindexlib.Machine),
+                                   ("distros", layerindexlib.Distro)]:
+                if lName in pindex:
+                    index.add_raw_element(lName, lType, pindex[lName])
+
+
+        if not os.path.isdir(up.path):
+            load_cache(up.path, index, branches)
+            return index
+
+        logger.debug(1, 'Loading from dir %s...' % (up.path))
+        for (dirpath, _, filenames) in os.walk(up.path):
+            for filename in filenames:
+                if not filename.endswith('.json'):
+                    continue
+                fpath = os.path.join(dirpath, filename)
+                load_cache(fpath, index, branches)
+
+        return index
+
+
+    def load_index_web(self, up, url, load):
+        """
+            Fetches layer information from a remote layer index.
+
+            The return value is a LayerIndexObj.
+
+            ud is the parsed url to the rest api of the layer index, such as:
+            http://layers.openembedded.org/layerindex/api/
+        """
+
+        def _get_json_response(apiurl=None, username=None, password=None, retry=True):
+            assert apiurl is not None
+
+            logger.debug(1, "fetching %s" % apiurl)
+
+            up = urlparse(apiurl)
+
+            username=up.username
+            password=up.password
+
+            # Strip username/password and params
+            if up.port:
+                up_stripped = up._replace(params="", netloc="%s:%s" % (up.hostname, up.port))
+            else:
+                up_stripped = up._replace(params="", netloc=up.hostname)
+
+            res = self.layerindex._fetch_url(up_stripped.geturl(), username=username, password=password)
+
+            try:
+                parsed = json.loads(res.read().decode('utf-8'))
+            except ConnectionResetError:
+                if retry:
+                    logger.debug(1, "%s: Connection reset by peer.  Retrying..." % url)
+                    parsed = _get_json_response(apiurl=up_stripped.geturl(), username=username, password=password, retry=False)
+                    logger.debug(1, "%s: retry successful.")
+                else:
+                    raise LayerIndexFetchError('%s: Connection reset by peer.  Is there a firewall blocking your connection?' % apiurl)
+
+            return parsed
+
+        index = layerindexlib.LayerIndexObj()
+
+        index.config = {}
+        index.config['TYPE'] = self.type
+        index.config['URL'] = url
+
+        params = self.layerindex._parse_params(up.params)
+
+        if 'desc' in params:
+            index.config['DESCRIPTION'] = unquote(params['desc'])
+        else:
+            index.config['DESCRIPTION'] = up.hostname
+
+        if 'cache' in params:
+            index.config['CACHE'] = params['cache']
+
+        if 'branch' in params:
+            branches = params['branch'].split(',')
+            index.config['BRANCH'] = branches
+        else:
+            branches = ['*']
+
+        try:
+            index.apilinks = _get_json_response(apiurl=url, username=up.username, password=up.password)
+        except Exception as e:
+            raise layerindexlib.LayerIndexFetchError(url, e)
+
+        # Local raw index set...
+        pindex = {}
+
+        # Load all the requested branches at the same time time,
+        # a special branch of '*' means load all branches
+        filter = ""
+        if "*" not in branches:
+            filter = "?filter=name:%s" % "OR".join(branches)
+
+        logger.debug(1, "Loading %s from %s" % (branches, index.apilinks['branches']))
+
+        # The link won't include username/password, so pull it from the original url
+        pindex['branches'] = _get_json_response(index.apilinks['branches'] + filter,
+                                                    username=up.username, password=up.password)
+        if not pindex['branches']:
+            logger.debug(1, "No valid branches (%s) found at url %s." % (branch, url))
+            return index
+        index.add_raw_element("branches", layerindexlib.Branch, pindex['branches'])
+
+        # Load all of the layerItems (these can not be easily filtered)
+        logger.debug(1, "Loading %s from %s" % ('layerItems', index.apilinks['layerItems']))
+
+
+        # The link won't include username/password, so pull it from the original url
+        pindex['layerItems'] = _get_json_response(index.apilinks['layerItems'],
+                                                  username=up.username, password=up.password)
+        if not pindex['layerItems']:
+            logger.debug(1, "No layers were found at url %s." % (url))
+            return index
+        index.add_raw_element("layerItems", layerindexlib.LayerItem, pindex['layerItems'])
+
+
+	# From this point on load the contents for each branch.  Otherwise we
+	# could run into a timeout.
+        for branch in index.branches:
+            filter = "?filter=branch__name:%s" % index.branches[branch].name
+
+            logger.debug(1, "Loading %s from %s" % ('layerBranches', index.apilinks['layerBranches']))
+
+            # The link won't include username/password, so pull it from the original url
+            pindex['layerBranches'] = _get_json_response(index.apilinks['layerBranches'] + filter,
+                                                  username=up.username, password=up.password)
+            if not pindex['layerBranches']:
+                logger.debug(1, "No valid layer branches (%s) found at url %s." % (branches or "*", url))
+                return index
+            index.add_raw_element("layerBranches", layerindexlib.LayerBranch, pindex['layerBranches'])
+
+
+            # Load the rest, they all have a similar format
+            # Note: the layer index has a few more items, we can add them if necessary
+            # in the future.
+            filter = "?filter=layerbranch__branch__name:%s" % index.branches[branch].name
+            for (lName, lType) in [("layerDependencies", layerindexlib.LayerDependency),
+                                   ("recipes", layerindexlib.Recipe),
+                                   ("machines", layerindexlib.Machine),
+                                   ("distros", layerindexlib.Distro)]:
+                if lName not in load:
+                    continue
+                logger.debug(1, "Loading %s from %s" % (lName, index.apilinks[lName]))
+
+                # The link won't include username/password, so pull it from the original url
+                pindex[lName] = _get_json_response(index.apilinks[lName] + filter,
+                                            username=up.username, password=up.password)
+                index.add_raw_element(lName, lType, pindex[lName])
+
+        return index
+
+    def store_index(self, url, index):
+        """
+            Store layer information into a local file/dir.
+
+            The return value is a dictionary containing API,
+            layer, branch, dependency, recipe, machine, distro, information.
+
+            ud is a parsed url to a directory or file.  If the path is a
+            directory, we will split the files into one file per layer.
+            If the path is to a file (exists or not) the entire DB will be
+            dumped into that one file.
+        """
+
+        up = urlparse(url)
+
+        if up.scheme != 'file':
+            raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
+
+        logger.debug(1, "Storing to %s..." % up.path)
+
+        try:
+            layerbranches = index.layerBranches
+        except KeyError:
+            logger.error('No layerBranches to write.')
+            return
+
+
+        def filter_item(layerbranchid, objects):
+            filtered = []
+            for obj in getattr(index, objects, None):
+                try:
+                    if getattr(index, objects)[obj].layerbranch_id == layerbranchid:
+                       filtered.append(getattr(index, objects)[obj]._data)
+                except AttributeError:
+                    logger.debug(1, 'No obj.layerbranch_id: %s' % objects)
+                    # No simple filter method, just include it...
+                    try:
+                        filtered.append(getattr(index, objects)[obj]._data)
+                    except AttributeError:
+                        logger.debug(1, 'No obj._data: %s %s' % (objects, type(obj)))
+                        filtered.append(obj)
+            return filtered
+
+
+        # Write out to a single file.
+        # Filter out unnecessary items, then sort as we write for determinism
+        if not os.path.isdir(up.path):
+            pindex = {}
+
+            pindex['branches'] = []
+            pindex['layerItems'] = []
+            pindex['layerBranches'] = []
+
+            for layerbranchid in layerbranches:
+                if layerbranches[layerbranchid].branch._data not in pindex['branches']:
+                    pindex['branches'].append(layerbranches[layerbranchid].branch._data)
+
+                if layerbranches[layerbranchid].layer._data not in pindex['layerItems']:
+                    pindex['layerItems'].append(layerbranches[layerbranchid].layer._data)
+
+                if layerbranches[layerbranchid]._data not in pindex['layerBranches']:
+                    pindex['layerBranches'].append(layerbranches[layerbranchid]._data)
+
+                for entry in index._index:
+                    # Skip local items, apilinks and items already processed
+                    if entry in index.config['local'] or \
+                       entry == 'apilinks' or \
+                       entry == 'branches' or \
+                       entry == 'layerBranches' or \
+                       entry == 'layerItems':
+                        continue
+                    if entry not in pindex:
+                        pindex[entry] = []
+                    pindex[entry].extend(filter_item(layerbranchid, entry))
+
+            bb.debug(1, 'Writing index to %s' % up.path)
+            with open(up.path, 'wt') as f:
+                json.dump(layerindexlib.sort_entry(pindex), f, indent=4)
+            return
+
+
+        # Write out to a directory one file per layerBranch
+        # Prepare all layer related items, to create a minimal file.
+        # We have to sort the entries as we write so they are deterministic
+        for layerbranchid in layerbranches:
+            pindex = {}
+
+            for entry in index._index:
+                # Skip local items, apilinks and items already processed
+                if entry in index.config['local'] or \
+                   entry == 'apilinks' or \
+                   entry == 'branches' or \
+                   entry == 'layerBranches' or \
+                   entry == 'layerItems':
+                    continue
+                pindex[entry] = filter_item(layerbranchid, entry)
+
+            # Add the layer we're processing as the first one...
+            pindex['branches'] = [layerbranches[layerbranchid].branch._data]
+            pindex['layerItems'] = [layerbranches[layerbranchid].layer._data]
+            pindex['layerBranches'] = [layerbranches[layerbranchid]._data]
+
+            # We also need to include the layerbranch for any dependencies...
+            for layerdep in pindex['layerDependencies']:
+                layerdependency = layerindexlib.LayerDependency(index, layerdep)
+
+                layeritem = layerdependency.dependency
+                layerbranch = layerdependency.dependency_layerBranch
+
+                # We need to avoid duplicates...
+                if layeritem._data not in pindex['layerItems']:
+                    pindex['layerItems'].append(layeritem._data)
+
+                if layerbranch._data not in pindex['layerBranches']:
+                    pindex['layerBranches'].append(layerbranch._data)
+
+            # apply mirroring adjustments here....
+
+            fname = index.config['DESCRIPTION'] + '__' + pindex['branches'][0]['name'] + '__' + pindex['layerItems'][0]['name']
+            fname = fname.translate(str.maketrans('/ ', '__'))
+            fpath = os.path.join(up.path, fname)
+
+            bb.debug(1, 'Writing index to %s' % fpath + '.json')
+            with open(fpath + '.json', 'wt') as f:
+                json.dump(layerindexlib.sort_entry(pindex), f, indent=4)
diff --git a/lib/layerindexlib/tests/__init__.py b/lib/layerindexlib/tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/lib/layerindexlib/tests/common.py b/lib/layerindexlib/tests/common.py
new file mode 100644
index 0000000..22a5458
--- /dev/null
+++ b/lib/layerindexlib/tests/common.py
@@ -0,0 +1,43 @@
+# Copyright (C) 2017-2018 Wind River Systems, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import unittest
+import tempfile
+import os
+import bb
+
+import logging
+
+class LayersTest(unittest.TestCase):
+
+    def setUp(self):
+        self.origdir = os.getcwd()
+        self.d = bb.data.init()
+        # At least one variable needs to be set
+        self.d.setVar('DL_DIR', os.getcwd())
+
+        if os.environ.get("BB_SKIP_NETTESTS") == "yes":
+            self.d.setVar('BB_NO_NETWORK', '1')
+
+        self.tempdir = tempfile.mkdtemp()
+        self.logger = logging.getLogger("BitBake")
+
+    def tearDown(self):
+        os.chdir(self.origdir)
+        if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
+            print("Not cleaning up %s. Please remove manually." % self.tempdir)
+        else:
+            bb.utils.prunedir(self.tempdir)
+
diff --git a/lib/layerindexlib/tests/cooker.py b/lib/layerindexlib/tests/cooker.py
new file mode 100644
index 0000000..9ce6e8c
--- /dev/null
+++ b/lib/layerindexlib/tests/cooker.py
@@ -0,0 +1,123 @@
+# Copyright (C) 2018 Wind River Systems, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import unittest
+import tempfile
+import os
+import bb
+
+import layerindexlib
+from layerindexlib.tests.common import LayersTest
+
+import logging
+
+class LayerIndexCookerTest(LayersTest):
+
+    def setUp(self):
+        LayersTest.setUp(self)
+
+        # Note this is NOT a comprehensive test of cooker, as we can't easily
+        # configure the test data.  But we can emulate the basics of the layer.conf
+        # files, so that is what we will do.
+
+        new_topdir = os.path.join(os.path.dirname(__file__), "testdata")
+        new_bbpath = os.path.join(new_topdir, "build")
+
+        self.d.setVar('TOPDIR', new_topdir)
+        self.d.setVar('BBPATH', new_bbpath)
+
+        self.d = bb.parse.handle("%s/conf/bblayers.conf" % new_bbpath, self.d, True)
+        for layer in self.d.getVar('BBLAYERS').split():
+            self.d = bb.parse.handle("%s/conf/layer.conf" % layer, self.d, True)
+
+        self.layerindex = layerindexlib.LayerIndex(self.d)
+        self.layerindex.load_layerindex('cooker://', load=['layerDependencies'])
+
+    def test_layerindex_is_empty(self):
+        self.assertFalse(self.layerindex.is_empty(), msg="Layerindex is not empty!")
+
+    def test_dependency_resolution(self):
+        # Verify depth first searching...
+        (dependencies, invalidnames) = self.layerindex.find_dependencies(names=['meta-python'])
+
+        first = True
+        for deplayerbranch in dependencies:
+            layerBranch = dependencies[deplayerbranch][0]
+            layerDeps = dependencies[deplayerbranch][1:]
+
+            if not first:
+                continue
+
+            first = False
+
+            # Top of the deps should be openembedded-core, since everything depends on it.
+            self.assertEqual(layerBranch.layer.name, "openembedded-core", msg='Top dependency not openembedded-core')
+
+            # meta-python should cause an openembedded-core dependency, if not assert!
+            for dep in layerDeps:
+                if dep.layer.name == 'meta-python':
+                    break
+            else:
+                self.assertTrue(False, msg='meta-python was not found')
+
+            # Only check the first element...
+            break
+        else:
+            if first:
+                # Empty list, this is bad.
+                self.assertTrue(False, msg='Empty list of dependencies')
+
+            # Last dep should be the requested item
+            layerBranch = dependencies[deplayerbranch][0]
+            self.assertEqual(layerBranch.layer.name, "meta-python", msg='Last dependency not meta-python')
+
+    def test_find_collection(self):
+        def _check(collection, expected):
+            self.logger.debug(1, "Looking for collection %s..." % collection)
+            result = self.layerindex.find_collection(collection)
+            if expected:
+                self.assertIsNotNone(result, msg="Did not find %s when it shouldn't be there" % collection)
+            else:
+                self.assertIsNone(result, msg="Found %s when it should be there" % collection)
+
+        tests = [ ('core', True),
+                  ('openembedded-core', False),
+                  ('networking-layer', True),
+                  ('meta-python', True),
+                  ('openembedded-layer', True),
+                  ('notpresent', False) ]
+
+        for collection,result in tests:
+            _check(collection, result)
+
+    def test_find_layerbranch(self):
+        def _check(name, expected):
+            self.logger.debug(1, "Looking for layerbranch %s..." % name)
+            result = self.layerindex.find_layerbranch(name)
+            if expected:
+                self.assertIsNotNone(result, msg="Did not find %s when it shouldn't be there" % collection)
+            else:
+                self.assertIsNone(result, msg="Found %s when it should be there" % collection)
+
+        tests = [ ('openembedded-core', True),
+                  ('core', False),
+                  ('networking-layer', True),
+                  ('meta-python', True),
+                  ('openembedded-layer', True),
+                  ('notpresent', False) ]
+
+        for collection,result in tests:
+            _check(collection, result)
+
diff --git a/lib/layerindexlib/tests/layerindexobj.py b/lib/layerindexlib/tests/layerindexobj.py
new file mode 100644
index 0000000..e2fbb95
--- /dev/null
+++ b/lib/layerindexlib/tests/layerindexobj.py
@@ -0,0 +1,226 @@
+# Copyright (C) 2017-2018 Wind River Systems, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import unittest
+import tempfile
+import os
+import bb
+
+from layerindexlib.tests.common import LayersTest
+
+import logging
+
+class LayerIndexObjectsTest(LayersTest):
+    def setUp(self):
+        from layerindexlib import LayerIndexObj, Branch, LayerItem, LayerBranch, LayerDependency, Recipe, Machine, Distro
+
+        LayersTest.setUp(self)
+
+        self.index = LayerIndexObj()
+
+        branchId = 0
+        layerItemId = 0
+        layerBranchId = 0
+        layerDependencyId = 0
+        recipeId = 0
+        machineId = 0
+        distroId = 0
+
+        self.index.branches = {}
+        self.index.layerItems = {}
+        self.index.layerBranches = {}
+        self.index.layerDependencies = {}
+        self.index.recipes = {}
+        self.index.machines = {}
+        self.index.distros = {}
+
+        branchId += 1
+        self.index.branches[branchId] = Branch(self.index)
+        self.index.branches[branchId].define_data(branchId,
+                                        'test_branch', 'bb_test_branch')
+        self.index.branches[branchId].lockData()
+
+        layerItemId +=1
+        self.index.layerItems[layerItemId] = LayerItem(self.index)
+        self.index.layerItems[layerItemId].define_data(layerItemId,
+                                        'test_layerItem', vcs_url='git://git_test_url/test_layerItem')
+        self.index.layerItems[layerItemId].lockData()
+
+        layerBranchId +=1
+        self.index.layerBranches[layerBranchId] = LayerBranch(self.index)
+        self.index.layerBranches[layerBranchId].define_data(layerBranchId,
+                                        'test_collection', '99', layerItemId,
+                                        branchId)
+
+        recipeId += 1
+        self.index.recipes[recipeId] = Recipe(self.index)
+        self.index.recipes[recipeId].define_data(recipeId, 'test_git.bb',
+                                        'recipes-test', 'test', 'git',
+                                        layerBranchId)
+
+        machineId += 1
+        self.index.machines[machineId] = Machine(self.index)
+        self.index.machines[machineId].define_data(machineId,
+                                        'test_machine', 'test_machine',
+                                        layerBranchId)
+
+        distroId += 1
+        self.index.distros[distroId] = Distro(self.index)
+        self.index.distros[distroId].define_data(distroId,
+                                        'test_distro', 'test_distro',
+                                        layerBranchId)
+
+        layerItemId +=1
+        self.index.layerItems[layerItemId] = LayerItem(self.index)
+        self.index.layerItems[layerItemId].define_data(layerItemId, 'test_layerItem 2',
+                                        vcs_url='git://git_test_url/test_layerItem')
+
+        layerBranchId +=1
+        self.index.layerBranches[layerBranchId] = LayerBranch(self.index)
+        self.index.layerBranches[layerBranchId].define_data(layerBranchId,
+                                        'test_collection_2', '72', layerItemId,
+                                        branchId, actual_branch='some_other_branch')
+
+        layerDependencyId += 1
+        self.index.layerDependencies[layerDependencyId] = LayerDependency(self.index)
+        self.index.layerDependencies[layerDependencyId].define_data(layerDependencyId,
+                                        layerBranchId, 1)
+
+        layerDependencyId += 1
+        self.index.layerDependencies[layerDependencyId] = LayerDependency(self.index)
+        self.index.layerDependencies[layerDependencyId].define_data(layerDependencyId,
+                                        layerBranchId, 1, required=False)
+
+    def test_branch(self):
+        branch = self.index.branches[1]
+        self.assertEqual(branch.id, 1)
+        self.assertEqual(branch.name, 'test_branch')
+        self.assertEqual(branch.short_description, 'test_branch')
+        self.assertEqual(branch.bitbake_branch, 'bb_test_branch')
+
+    def test_layerItem(self):
+        layerItem = self.index.layerItems[1]
+        self.assertEqual(layerItem.id, 1)
+        self.assertEqual(layerItem.name, 'test_layerItem')
+        self.assertEqual(layerItem.summary, 'test_layerItem')
+        self.assertEqual(layerItem.description, 'test_layerItem')
+        self.assertEqual(layerItem.vcs_url, 'git://git_test_url/test_layerItem')
+        self.assertEqual(layerItem.vcs_web_url, None)
+        self.assertIsNone(layerItem.vcs_web_tree_base_url)
+        self.assertIsNone(layerItem.vcs_web_file_base_url)
+        self.assertIsNotNone(layerItem.updated)
+
+        layerItem = self.index.layerItems[2]
+        self.assertEqual(layerItem.id, 2)
+        self.assertEqual(layerItem.name, 'test_layerItem 2')
+        self.assertEqual(layerItem.summary, 'test_layerItem 2')
+        self.assertEqual(layerItem.description, 'test_layerItem 2')
+        self.assertEqual(layerItem.vcs_url, 'git://git_test_url/test_layerItem')
+        self.assertIsNone(layerItem.vcs_web_url)
+        self.assertIsNone(layerItem.vcs_web_tree_base_url)
+        self.assertIsNone(layerItem.vcs_web_file_base_url)
+        self.assertIsNotNone(layerItem.updated)
+
+    def test_layerBranch(self):
+        layerBranch = self.index.layerBranches[1]
+        self.assertEqual(layerBranch.id, 1)
+        self.assertEqual(layerBranch.collection, 'test_collection')
+        self.assertEqual(layerBranch.version, '99')
+        self.assertEqual(layerBranch.vcs_subdir, '')
+        self.assertEqual(layerBranch.actual_branch, 'test_branch')
+        self.assertIsNotNone(layerBranch.updated)
+        self.assertEqual(layerBranch.layer_id, 1)
+        self.assertEqual(layerBranch.branch_id, 1)
+        self.assertEqual(layerBranch.layer, self.index.layerItems[1])
+        self.assertEqual(layerBranch.branch, self.index.branches[1])
+
+        layerBranch = self.index.layerBranches[2]
+        self.assertEqual(layerBranch.id, 2)
+        self.assertEqual(layerBranch.collection, 'test_collection_2')
+        self.assertEqual(layerBranch.version, '72')
+        self.assertEqual(layerBranch.vcs_subdir, '')
+        self.assertEqual(layerBranch.actual_branch, 'some_other_branch')
+        self.assertIsNotNone(layerBranch.updated)
+        self.assertEqual(layerBranch.layer_id, 2)
+        self.assertEqual(layerBranch.branch_id, 1)
+        self.assertEqual(layerBranch.layer, self.index.layerItems[2])
+        self.assertEqual(layerBranch.branch, self.index.branches[1])
+
+    def test_layerDependency(self):
+        layerDependency = self.index.layerDependencies[1]
+        self.assertEqual(layerDependency.id, 1)
+        self.assertEqual(layerDependency.layerbranch_id, 2)
+        self.assertEqual(layerDependency.layerbranch, self.index.layerBranches[2])
+        self.assertEqual(layerDependency.layer_id, 2)
+        self.assertEqual(layerDependency.layer, self.index.layerItems[2])
+        self.assertTrue(layerDependency.required)
+        self.assertEqual(layerDependency.dependency_id, 1)
+        self.assertEqual(layerDependency.dependency, self.index.layerItems[1])
+        self.assertEqual(layerDependency.dependency_layerBranch, self.index.layerBranches[1])
+
+        layerDependency = self.index.layerDependencies[2]
+        self.assertEqual(layerDependency.id, 2)
+        self.assertEqual(layerDependency.layerbranch_id, 2)
+        self.assertEqual(layerDependency.layerbranch, self.index.layerBranches[2])
+        self.assertEqual(layerDependency.layer_id, 2)
+        self.assertEqual(layerDependency.layer, self.index.layerItems[2])
+        self.assertFalse(layerDependency.required)
+        self.assertEqual(layerDependency.dependency_id, 1)
+        self.assertEqual(layerDependency.dependency, self.index.layerItems[1])
+        self.assertEqual(layerDependency.dependency_layerBranch, self.index.layerBranches[1])
+
+    def test_recipe(self):
+        recipe = self.index.recipes[1]
+        self.assertEqual(recipe.id, 1)
+        self.assertEqual(recipe.layerbranch_id, 1)
+        self.assertEqual(recipe.layerbranch, self.index.layerBranches[1])
+        self.assertEqual(recipe.layer_id, 1)
+        self.assertEqual(recipe.layer, self.index.layerItems[1])
+        self.assertEqual(recipe.filename, 'test_git.bb')
+        self.assertEqual(recipe.filepath, 'recipes-test')
+        self.assertEqual(recipe.fullpath, 'recipes-test/test_git.bb')
+        self.assertEqual(recipe.summary, "")
+        self.assertEqual(recipe.description, "")
+        self.assertEqual(recipe.section, "")
+        self.assertEqual(recipe.pn, 'test')
+        self.assertEqual(recipe.pv, 'git')
+        self.assertEqual(recipe.license, "")
+        self.assertEqual(recipe.homepage, "")
+        self.assertEqual(recipe.bugtracker, "")
+        self.assertEqual(recipe.provides, "")
+        self.assertIsNotNone(recipe.updated)
+        self.assertEqual(recipe.inherits, "")
+
+    def test_machine(self):
+        machine = self.index.machines[1]
+        self.assertEqual(machine.id, 1)
+        self.assertEqual(machine.layerbranch_id, 1)
+        self.assertEqual(machine.layerbranch, self.index.layerBranches[1])
+        self.assertEqual(machine.layer_id, 1)
+        self.assertEqual(machine.layer, self.index.layerItems[1])
+        self.assertEqual(machine.name, 'test_machine')
+        self.assertEqual(machine.description, 'test_machine')
+        self.assertIsNotNone(machine.updated)
+
+    def test_distro(self):
+        distro = self.index.distros[1]
+        self.assertEqual(distro.id, 1)
+        self.assertEqual(distro.layerbranch_id, 1)
+        self.assertEqual(distro.layerbranch, self.index.layerBranches[1])
+        self.assertEqual(distro.layer_id, 1)
+        self.assertEqual(distro.layer, self.index.layerItems[1])
+        self.assertEqual(distro.name, 'test_distro')
+        self.assertEqual(distro.description, 'test_distro')
+        self.assertIsNotNone(distro.updated)
diff --git a/lib/layerindexlib/tests/restapi.py b/lib/layerindexlib/tests/restapi.py
new file mode 100644
index 0000000..bfaac43
--- /dev/null
+++ b/lib/layerindexlib/tests/restapi.py
@@ -0,0 +1,174 @@
+# Copyright (C) 2017-2018 Wind River Systems, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import unittest
+import tempfile
+import os
+import bb
+
+import layerindexlib
+from layerindexlib.tests.common import LayersTest
+
+import logging
+
+class LayerIndexWebRestApiTest(LayersTest):
+
+    if os.environ.get("BB_SKIP_NETTESTS") == "yes":
+        print("Unset BB_SKIP_NETTESTS to run network tests")
+    else:
+        def setUp(self):
+            LayersTest.setUp(self)
+            self.layerindex = layerindexlib.LayerIndex(self.d)
+            self.layerindex.load_layerindex('http://layers.openembedded.org/layerindex/api/;branch=sumo', load=['layerDependencies'])
+
+        def test_layerindex_is_empty(self):
+            self.assertFalse(self.layerindex.is_empty(), msg="Layerindex is empty")
+
+        def test_layerindex_store_file(self):
+            self.layerindex.store_layerindex('file://%s/file.json' % self.tempdir, self.layerindex.indexes[0])
+
+            self.assertTrue(os.path.isfile('%s/file.json' % self.tempdir), msg="Temporary file was not created by store_layerindex")
+
+            reload = layerindexlib.LayerIndex(self.d)
+            reload.load_layerindex('file://%s/file.json' % self.tempdir)
+
+            self.assertFalse(reload.is_empty(), msg="Layerindex is empty")
+
+            # Calculate layerItems in original index that should NOT be in reload
+            layerItemNames = []
+            for itemId in self.layerindex.indexes[0].layerItems:
+                layerItemNames.append(self.layerindex.indexes[0].layerItems[itemId].name)
+
+            for layerBranchId in self.layerindex.indexes[0].layerBranches:
+                layerItemNames.remove(self.layerindex.indexes[0].layerBranches[layerBranchId].layer.name)
+
+            for itemId in reload.indexes[0].layerItems:
+                self.assertFalse(reload.indexes[0].layerItems[itemId].name in layerItemNames, msg="Item reloaded when it shouldn't have been")
+
+            # Compare the original to what we wrote...
+            for type in self.layerindex.indexes[0]._index:
+                if type == 'apilinks' or \
+                   type == 'layerItems' or \
+                   type in self.layerindex.indexes[0].config['local']:
+                    continue
+                for id in getattr(self.layerindex.indexes[0], type):
+                    self.logger.debug(1, "type %s" % (type))
+
+                    self.assertTrue(id in getattr(reload.indexes[0], type), msg="Id number not in reloaded index")
+
+                    self.logger.debug(1, "%s ? %s" % (getattr(self.layerindex.indexes[0], type)[id], getattr(reload.indexes[0], type)[id]))
+
+                    self.assertEqual(getattr(self.layerindex.indexes[0], type)[id], getattr(reload.indexes[0], type)[id], msg="Reloaded contents different")
+
+        def test_layerindex_store_split(self):
+            self.layerindex.store_layerindex('file://%s' % self.tempdir, self.layerindex.indexes[0])
+
+            reload = layerindexlib.LayerIndex(self.d)
+            reload.load_layerindex('file://%s' % self.tempdir)
+
+            self.assertFalse(reload.is_empty(), msg="Layer index is empty")
+
+            for type in self.layerindex.indexes[0]._index:
+                if type == 'apilinks' or \
+                   type == 'layerItems' or \
+                   type in self.layerindex.indexes[0].config['local']:
+                    continue
+                for id in getattr(self.layerindex.indexes[0] ,type):
+                    self.logger.debug(1, "type %s" % (type))
+
+                    self.assertTrue(id in getattr(reload.indexes[0], type), msg="Id number missing from reloaded data")
+
+                    self.logger.debug(1, "%s ? %s" % (getattr(self.layerindex.indexes[0] ,type)[id], getattr(reload.indexes[0], type)[id]))
+
+                    self.assertEqual(getattr(self.layerindex.indexes[0] ,type)[id], getattr(reload.indexes[0], type)[id], msg="reloaded data does not match original")
+
+        def test_dependency_resolution(self):
+            # Verify depth first searching...
+            (dependencies, invalidnames) = self.layerindex.find_dependencies(names=['meta-python'])
+
+            first = True
+            for deplayerbranch in dependencies:
+                layerBranch = dependencies[deplayerbranch][0]
+                layerDeps = dependencies[deplayerbranch][1:]
+
+                if not first:
+                    continue
+
+                first = False
+
+                # Top of the deps should be openembedded-core, since everything depends on it.
+                self.assertEqual(layerBranch.layer.name, "openembedded-core", msg='OpenEmbedded-Core is no the first dependency')
+
+                # meta-python should cause an openembedded-core dependency, if not assert!
+                for dep in layerDeps:
+                    if dep.layer.name == 'meta-python':
+                        break
+                else:
+                    self.logger.debug(1, "meta-python was not found")
+                    self.assetTrue(False)
+
+                # Only check the first element...
+                break
+            else:
+                # Empty list, this is bad.
+                self.logger.debug(1, "Empty list of dependencies")
+                self.assertIsNotNone(first, msg="Empty list of dependencies")
+
+                # Last dep should be the requested item
+                layerBranch = dependencies[deplayerbranch][0]
+                self.assertEqual(layerBranch.layer.name, "meta-python", msg="Last dependency not meta-python")
+
+    def test_find_collection(self):
+        def _check(collection, expected):
+            self.logger.debug(1, "Looking for collection %s..." % collection)
+            result = self.layerindex.find_collection(collection)
+            if expected:
+                self.assertIsNotNone(result, msg="Did not find %s when it should be there" % collection)
+            else:
+                self.assertIsNone(result, msg="Found %s when it shouldn't be there" % collection)
+
+        tests = [ ('core', True),
+                  ('openembedded-core', False),
+                  ('networking-layer', True),
+                  ('meta-python', True),
+                  ('openembedded-layer', True),
+                  ('notpresent', False) ]
+
+        for collection,result in tests:
+            _check(collection, result)
+
+    def test_find_layerbranch(self):
+        def _check(name, expected):
+            self.logger.debug(1, "Looking for layerbranch %s..." % name)
+
+            for index in self.layerindex.indexes:
+                for layerbranchid in index.layerBranches:
+                    self.logger.debug(1, "Present: %s" % index.layerBranches[layerbranchid].layer.name)
+            result = self.layerindex.find_layerbranch(name)
+            if expected:
+                self.assertIsNotNone(result, msg="Did not find %s when it should be there" % collection)
+            else:
+                self.assertIsNone(result, msg="Found %s when it shouldn't be there" % collection)
+
+        tests = [ ('openembedded-core', True),
+                  ('core', False),
+                  ('meta-networking', True),
+                  ('meta-python', True),
+                  ('meta-oe', True),
+                  ('notpresent', False) ]
+
+        for collection,result in tests:
+            _check(collection, result)
+
diff --git a/lib/layerindexlib/tests/testdata/README b/lib/layerindexlib/tests/testdata/README
new file mode 100644
index 0000000..36ab40b
--- /dev/null
+++ b/lib/layerindexlib/tests/testdata/README
@@ -0,0 +1,11 @@
+This test data is used to verify the 'cooker' module of the layerindex.
+
+The module consists of a faux project bblayers.conf with four layers defined.
+
+layer1 - openembedded-core
+layer2 - networking-layer
+layer3 - meta-python
+layer4 - openembedded-layer (meta-oe)
+
+Since we do not have a fully populated cooker, we use this to test the
+basic index generation, and not any deep recipe based contents.
diff --git a/lib/layerindexlib/tests/testdata/build/conf/bblayers.conf b/lib/layerindexlib/tests/testdata/build/conf/bblayers.conf
new file mode 100644
index 0000000..40429b2
--- /dev/null
+++ b/lib/layerindexlib/tests/testdata/build/conf/bblayers.conf
@@ -0,0 +1,15 @@
+LAYERSERIES_CORENAMES = "sumo"
+
+# LAYER_CONF_VERSION is increased each time build/conf/bblayers.conf
+# changes incompatibly
+LCONF_VERSION = "7"
+
+BBPATH = "${TOPDIR}"
+BBFILES ?= ""
+
+BBLAYERS ?= " \
+  ${TOPDIR}/layer1 \
+  ${TOPDIR}/layer2 \
+  ${TOPDIR}/layer3 \
+  ${TOPDIR}/layer4 \
+  "
diff --git a/lib/layerindexlib/tests/testdata/layer1/conf/layer.conf b/lib/layerindexlib/tests/testdata/layer1/conf/layer.conf
new file mode 100644
index 0000000..966d531
--- /dev/null
+++ b/lib/layerindexlib/tests/testdata/layer1/conf/layer.conf
@@ -0,0 +1,17 @@
+# We have a conf and classes directory, add to BBPATH
+BBPATH .= ":${LAYERDIR}"
+# We have recipes-* directories, add to BBFILES
+BBFILES += "${LAYERDIR}/recipes-*/*/*.bb"
+
+BBFILE_COLLECTIONS += "core"
+BBFILE_PATTERN_core = "^${LAYERDIR}/"
+BBFILE_PRIORITY_core = "5"
+
+LAYERSERIES_CORENAMES = "sumo"
+
+# This should only be incremented on significant changes that will
+# cause compatibility issues with other layers
+LAYERVERSION_core = "11"
+LAYERSERIES_COMPAT_core = "sumo"
+
+BBLAYERS_LAYERINDEX_NAME_core = "openembedded-core"
diff --git a/lib/layerindexlib/tests/testdata/layer2/conf/layer.conf b/lib/layerindexlib/tests/testdata/layer2/conf/layer.conf
new file mode 100644
index 0000000..7569d1c
--- /dev/null
+++ b/lib/layerindexlib/tests/testdata/layer2/conf/layer.conf
@@ -0,0 +1,20 @@
+# We have a conf and classes directory, add to BBPATH
+BBPATH .= ":${LAYERDIR}"
+
+# We have a packages directory, add to BBFILES
+BBFILES += "${LAYERDIR}/recipes-*/*/*.bb \
+            ${LAYERDIR}/recipes-*/*/*.bbappend"
+
+BBFILE_COLLECTIONS += "networking-layer"
+BBFILE_PATTERN_networking-layer := "^${LAYERDIR}/"
+BBFILE_PRIORITY_networking-layer = "5"
+
+# This should only be incremented on significant changes that will
+# cause compatibility issues with other layers
+LAYERVERSION_networking-layer = "1"
+
+LAYERDEPENDS_networking-layer = "core"
+LAYERDEPENDS_networking-layer += "openembedded-layer"
+LAYERDEPENDS_networking-layer += "meta-python"
+
+LAYERSERIES_COMPAT_networking-layer = "sumo"
diff --git a/lib/layerindexlib/tests/testdata/layer3/conf/layer.conf b/lib/layerindexlib/tests/testdata/layer3/conf/layer.conf
new file mode 100644
index 0000000..7089071
--- /dev/null
+++ b/lib/layerindexlib/tests/testdata/layer3/conf/layer.conf
@@ -0,0 +1,19 @@
+# We might have a conf and classes directory, append to BBPATH
+BBPATH .= ":${LAYERDIR}"
+
+# We have recipes directories, add to BBFILES
+BBFILES += "${LAYERDIR}/recipes*/*/*.bb ${LAYERDIR}/recipes*/*/*.bbappend"
+
+BBFILE_COLLECTIONS += "meta-python"
+BBFILE_PATTERN_meta-python := "^${LAYERDIR}/"
+BBFILE_PRIORITY_meta-python = "7"
+
+# This should only be incremented on significant changes that will
+# cause compatibility issues with other layers
+LAYERVERSION_meta-python = "1"
+
+LAYERDEPENDS_meta-python = "core openembedded-layer"
+
+LAYERSERIES_COMPAT_meta-python = "sumo"
+
+LICENSE_PATH += "${LAYERDIR}/licenses"
diff --git a/lib/layerindexlib/tests/testdata/layer4/conf/layer.conf b/lib/layerindexlib/tests/testdata/layer4/conf/layer.conf
new file mode 100644
index 0000000..6649ee0
--- /dev/null
+++ b/lib/layerindexlib/tests/testdata/layer4/conf/layer.conf
@@ -0,0 +1,22 @@
+# We have a conf and classes directory, append to BBPATH
+BBPATH .= ":${LAYERDIR}"
+
+# We have a recipes directory, add to BBFILES
+BBFILES += "${LAYERDIR}/recipes-*/*/*.bb ${LAYERDIR}/recipes-*/*/*.bbappend"
+
+BBFILE_COLLECTIONS += "openembedded-layer"
+BBFILE_PATTERN_openembedded-layer := "^${LAYERDIR}/"
+
+# Define the priority for recipes (.bb files) from this layer,
+# choosing carefully how this layer interacts with all of the
+# other layers.
+
+BBFILE_PRIORITY_openembedded-layer = "6"
+
+# This should only be incremented on significant changes that will
+# cause compatibility issues with other layers
+LAYERVERSION_openembedded-layer = "1"
+
+LAYERDEPENDS_openembedded-layer = "core"
+
+LAYERSERIES_COMPAT_openembedded-layer = "sumo"
-- 
1.8.3.1



^ permalink raw reply related

* [PATCH 1/5 v2] bblayers/layerindex.py: Fix addition of layers
From: Mark Hatle @ 2018-07-24  2:29 UTC (permalink / raw)
  To: bitbake-devel
In-Reply-To: <20180724022914.185634-1-mark.hatle@windriver.com>

When a layer is added it needs to be in a list, otherwise the system will
error such as:

    Specified layer directory / doesn't contain a conf/layer.conf file

Additionally, instead of calling the add layer function over and over, it
is better to add all of the new content in one command.  Otherwise the
order is important as the system now checks if the layer can be added.  For
instance, trying to add meta-python:

   Layer                Required by          Git repository                                          Subdirectory
   ===================================================================================================================
   meta-python          -                    git://git.openembedded.org/meta-openembedded            meta-python
   meta-oe              meta-python          git://git.openembedded.org/meta-openembedded            meta-oe
   openembedded-core    meta-python          git://git.openembedded.org/openembedded-core            meta
   Adding layer "meta-python" (.../oe-core/meta-openembedded/meta-python) to conf/bblayers.conf
   ERROR: Layer 'meta-python' depends on layer 'openembedded-layer', but this layer is not enabled in your configuration

The system would try to add meta-python before the dependent meta-oe.  Adding
them both at the same time resolves this issue.

Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
---
 lib/bblayers/layerindex.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/lib/bblayers/layerindex.py b/lib/bblayers/layerindex.py
index 9af385d..53c858d 100644
--- a/lib/bblayers/layerindex.py
+++ b/lib/bblayers/layerindex.py
@@ -239,19 +239,22 @@ class LayerIndexPlugin(ActionPlugin):
                     return 1
                 addlayers.append((subdir, name, layerdir))
         if not args.show_only:
+            localargs = argparse.Namespace()
+            localargs.layerdir = []
+            localargs.force = args.force
             for subdir, name, layerdir in set(addlayers):
                 if os.path.exists(layerdir):
                     if subdir:
-                        logger.plain("Adding layer \"%s\" to conf/bblayers.conf" % subdir)
+                        logger.plain("Adding layer \"%s\" (%s) to conf/bblayers.conf" % (subdir, layerdir))
                     else:
-                        logger.plain("Adding layer \"%s\" to conf/bblayers.conf" % name)
-                    localargs = argparse.Namespace()
-                    localargs.layerdir = layerdir
-                    localargs.force = args.force
-                    self.do_add_layer(localargs)
+                        logger.plain("Adding layer \"%s\" (%s) to conf/bblayers.conf" % (name, layerdir))
+                    localargs.layerdir.append(layerdir)
                 else:
                     break
 
+            if localargs.layerdir:
+                self.do_add_layer(localargs)
+
     def do_layerindex_show_depends(self, args):
         """Find layer dependencies from layer index.
 """
-- 
1.8.3.1



^ permalink raw reply related

* Re: [RFC PATCH 05/10] iommu/vt-d: Setup DMA remapping for mediated devices
From: Lu Baolu @ 2018-07-24  2:29 UTC (permalink / raw)
  To: Liu, Yi L, Joerg Roedel, David Woodhouse, Alex Williamson,
	Kirti Wankhede
  Cc: Raj, Ashok, Kumar, Sanjay K, Pan, Jacob jun, Tian, Kevin,
	Sun, Yi Y, peterx@redhat.com, iommu@lists.linux-foundation.org,
	kvm@vger.kernel.org, linux-kernel@vger.kernel.org, Jacob Pan
In-Reply-To: <A2975661238FB949B60364EF0F2C257439CA1449@SHSMSX104.ccr.corp.intel.com>

Hi,

On 07/23/2018 12:44 PM, Liu, Yi L wrote:
>> From: Lu Baolu [mailto:baolu.lu@linux.intel.com]
>> Sent: Sunday, July 22, 2018 2:09 PM
>>
>> This configures the second level page table when external components request to
>> allocate a domain for a mediated device.
> I think it should be the time when adding a mediate device to the domain.

You are right. Will correct it.

>
>> Cc: Ashok Raj <ashok.raj@intel.com>
>> Cc: Jacob Pan <jacob.jun.pan@linux.intel.com>
>> Cc: Kevin Tian <kevin.tian@intel.com>
>> Cc: Liu Yi L <yi.l.liu@intel.com>
>> Signed-off-by: Sanjay Kumar <sanjay.k.kumar@intel.com>
>> Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
>> ---
>>  drivers/iommu/intel-iommu.c | 73
>> ++++++++++++++++++++++++++++++++++++++++-----
>>  1 file changed, 66 insertions(+), 7 deletions(-)
>>
>> diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index
>> 57ccfc4..b6e9ea8 100644
>> --- a/drivers/iommu/intel-iommu.c
>> +++ b/drivers/iommu/intel-iommu.c
>> @@ -2569,8 +2569,9 @@ static struct dmar_domain
>> *dmar_insert_one_dev_info(struct intel_iommu *iommu,
>>  	if (dev)
>>  		dev->archdata.iommu = info;
>>
>> -	if (dev && dev_is_pci(dev) && sm_supported(iommu)) {
>> -		bool pass_through;
>> +	if (dev && sm_supported(iommu)) {
>> +		bool pass_through = hw_pass_through &&
>> +				domain_type_is_si(domain);
>>
>>  		ret = intel_pasid_alloc_table(dev);
>>  		if (ret) {
>> @@ -2579,12 +2580,21 @@ static struct dmar_domain
>> *dmar_insert_one_dev_info(struct intel_iommu *iommu,
>>  			return NULL;
>>  		}
>>
>> -		/* Setup the PASID entry for requests without PASID: */
>> -		pass_through = hw_pass_through && domain_type_is_si(domain);
>>  		spin_lock(&iommu->lock);
>> -		intel_pasid_setup_second_level(iommu, domain, dev,
>> -					       PASID_RID2PASID,
>> -					       pass_through);
>> +
>> +		/* Setup the PASID entry for requests without PASID: */
>> +		if (dev_is_pci(dev))
>> +			intel_pasid_setup_second_level(iommu, domain, dev,
>> +						       PASID_RID2PASID,
> can domain->default_pasid be initialized to be PASID_RID2PASID if the domain
> is not a domain used by mediated device?

PASID_RID2PASID is 0 as well.


>
>> +						       pass_through);
>> +		/* Setup the PASID entry for mediated devices: */
>> +		else if (dev_is_mdev(dev))
>> +			intel_pasid_setup_second_level(iommu, domain, dev,
>> +						       domain->default_pasid,
>> +						       false);
>> +		else
>> +			pr_err("Unsupported device %s\n", dev_name(dev));
>> +
>>  		spin_unlock(&iommu->lock);
>>  	}
>>  	spin_unlock_irqrestore(&device_domain_lock, flags); @@ -4937,6 +4947,32
>> @@ static void domain_context_clear(struct intel_iommu *iommu, struct device
>> *dev)
>>  	pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb,
>> iommu);  }
>>
>> +static void
>> +iommu_flush_ext_iotlb(struct intel_iommu *iommu, u16 did, u32 pasid,
> Per latest VT-d spec, its new name is pasid-based-iotlb. Pls update the naming
> accordingly. :)

Okay.

Best regards,
Lu Baolu

>
> Regards,
> Yi Liu
>
>> +u64 gran) {
>> +	struct qi_desc desc;
>> +
>> +	desc.high = 0;
>> +	desc.low = QI_EIOTLB_PASID(pasid) | QI_EIOTLB_DID(did) |
>> +			QI_EIOTLB_GRAN(gran) | QI_EIOTLB_TYPE;
>> +
>> +	qi_submit_sync(&desc, iommu);
>> +}
>> +
>> +static void iommu_flush_pasid_dev_iotlb(struct intel_iommu *iommu,
>> +					struct device *dev, int sid, int pasid) {
>> +	struct qi_desc desc;
>> +	struct pci_dev *pdev = to_pci_dev(dev);
>> +	int qdep = pci_ats_queue_depth(pdev);
>> +
>> +	desc.low = QI_DEV_EIOTLB_PASID(pasid) | QI_DEV_EIOTLB_SID(sid) |
>> +			QI_DEV_EIOTLB_QDEP(qdep) | QI_DEIOTLB_TYPE;
>> +	desc.high = QI_DEV_EIOTLB_ADDR(-1ULL >> 1) | QI_DEV_EIOTLB_SIZE;
>> +
>> +	qi_submit_sync(&desc, iommu);
>> +}
>> +
>>  static void __dmar_remove_one_dev_info(struct device_domain_info *info)  {
>>  	struct intel_iommu *iommu;
>> @@ -4949,6 +4985,29 @@ static void __dmar_remove_one_dev_info(struct
>> device_domain_info *info)
>>
>>  	iommu = info->iommu;
>>
>> +	if (dev_is_mdev(info->dev)) {
>> +		struct dmar_domain *domain = info->domain;
>> +		int did = domain->iommu_did[iommu->seq_id];
>> +		int sid = info->bus << 8 | info->devfn;
>> +		struct device *dev = info->dev;
>> +
>> +		intel_pasid_clear_entry(dev, domain->default_pasid);
>> +
>> +		/* Flush IOTLB including PASID Cache: */
>> +		iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
>> +
>> +		/*
>> +		 * Flush EIOTLB. The only way to flush global mappings within
>> +		 * a PASID is to use QI_GRAN_ALL_ALL.
>> +		 */
>> +		iommu_flush_ext_iotlb(iommu, did, domain->default_pasid,
>> +				      QI_GRAN_ALL_ALL);
>> +
>> +		/* Flush Dev TLB: */
>> +		iommu_flush_pasid_dev_iotlb(iommu, dev_mdev_parent(dev), sid,
>> +					    domain->default_pasid);
>> +	}
>> +
>>  	if (info->dev) {
>>  		iommu_disable_dev_iotlb(info);
>>  		domain_context_clear(iommu, info->dev);
>> --
>> 2.7.4
>

^ permalink raw reply

* [PATCH 0/5 v2] Add a standard module for accessing the layerindex
From: Mark Hatle @ 2018-07-24  2:29 UTC (permalink / raw)
  To: bitbake-devel

Changes available at:
git://git.openembedded.org/bitbake-contrib bitbake-layerindex

v2:
Refactored the module to address Paul E's concerns:

* Treat LAYERSERIES_CORENAMES as a list of supported branches

* Remove type= stuff in plugins, now iterate through plugins until one
  of them accepts the data

* Fix incorrect exceptions, add additional exceptions as necessary

* Replace space separated lists with python lists.. (Few minor excptions,
  documented why where still necessary.)

* Refactor item and object classes:
  - Each LayerIndexItem now uses both decorators and properties to manage
    access to the private data
  - New LayerIndexObj class to handle a -single- layerindex, the main
    module now is a list of LayerIndexObjs

* Lots of variable name changes and cleanups

* Removed dummy API to implement a json query

* General refactoring to accomplish the above

* bitbake-layers layerindex-*
  - Now checks the cooker first, if the layer is already present skips
    going to the network

Selftest passes.
bitbake-layers layerindex-fetch and layerindex-show-depends pass

Note: as before the toaster change needs additional verication then
      what I'm current able to provide.  But it -should- work as patched.


V1:
In order to simply existing components, and add support to create some
new functionaly -- we need a common apporach for access the layerindex.

The class supports loading multilib layerindexes, but right now that
functionality is not being used by either bitbake-layers or the toaster.

There are a few 'TODO' items that remain in the code.  These are related
to either un-implemented, but planned functionality or to display stuff
in bitbake-layers.  I'm hoping that part of this review can discuss the
TODO items.


Mark Hatle (5):
  bblayers/layerindex.py: Fix addition of layers
  layerindexlib: Initial layer index processing module implementation
  bblayers/layerindex.py: Switch to use the new layerindexlib class
  bitbake-layers: disable parsing for layerindex commands
  toaster/orm/management/commands/lsupdates.py: Use new layerindexlib
    module

 bin/bitbake-selftest                               |    6 +-
 lib/bblayers/layerindex.py                         |  323 ++---
 lib/layerindexlib/README                           |   28 +
 lib/layerindexlib/__init__.py                      | 1364 ++++++++++++++++++++
 lib/layerindexlib/cooker.py                        |  341 +++++
 lib/layerindexlib/plugin.py                        |   60 +
 lib/layerindexlib/restapi.py                       |  398 ++++++
 lib/layerindexlib/tests/__init__.py                |    0
 lib/layerindexlib/tests/common.py                  |   43 +
 lib/layerindexlib/tests/cooker.py                  |  123 ++
 lib/layerindexlib/tests/layerindexobj.py           |  226 ++++
 lib/layerindexlib/tests/restapi.py                 |  174 +++
 lib/layerindexlib/tests/testdata/README            |   11 +
 .../tests/testdata/build/conf/bblayers.conf        |   15 +
 .../tests/testdata/layer1/conf/layer.conf          |   17 +
 .../tests/testdata/layer2/conf/layer.conf          |   20 +
 .../tests/testdata/layer3/conf/layer.conf          |   19 +
 .../tests/testdata/layer4/conf/layer.conf          |   22 +
 lib/toaster/orm/management/commands/lsupdates.py   |  216 ++--
 19 files changed, 3083 insertions(+), 323 deletions(-)
 create mode 100644 lib/layerindexlib/README
 create mode 100644 lib/layerindexlib/__init__.py
 create mode 100644 lib/layerindexlib/cooker.py
 create mode 100644 lib/layerindexlib/plugin.py
 create mode 100644 lib/layerindexlib/restapi.py
 create mode 100644 lib/layerindexlib/tests/__init__.py
 create mode 100644 lib/layerindexlib/tests/common.py
 create mode 100644 lib/layerindexlib/tests/cooker.py
 create mode 100644 lib/layerindexlib/tests/layerindexobj.py
 create mode 100644 lib/layerindexlib/tests/restapi.py
 create mode 100644 lib/layerindexlib/tests/testdata/README
 create mode 100644 lib/layerindexlib/tests/testdata/build/conf/bblayers.conf
 create mode 100644 lib/layerindexlib/tests/testdata/layer1/conf/layer.conf
 create mode 100644 lib/layerindexlib/tests/testdata/layer2/conf/layer.conf
 create mode 100644 lib/layerindexlib/tests/testdata/layer3/conf/layer.conf
 create mode 100644 lib/layerindexlib/tests/testdata/layer4/conf/layer.conf

-- 
1.8.3.1



^ permalink raw reply


This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.