[rpms/apptainer] migrate-spdx: update to 1.1.0-rc.3

[Dave Dykstra <2129743+DrDaveD@users.noreply.github.com>]

A new commit has been pushed.

Repo   : rpms/apptainer
Branch : migrate-spdx
Commit : a51a647e58fea04bc2010e027dbbd89ed95d394f
Author : Dave Dykstra <2129743+DrDaveD@users.noreply.github.com>
Date   : 2022-09-06T13:16:45-05:00
Stats  : +1847/-32 in 5 file(s)
URL    : https://src.fedoraproject.org/rpms/apptainer/c/a51a647e58fea04bc2010e027dbbd89ed95d394f?branch=migrate-spdx

Log:
update to 1.1.0-rc.3

---
diff --git a/.gitignore b/.gitignore
index 9892dd8..7414331 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
 /apptainer-*.tar.gz
+/squashfuse-0.1.105.tar.gz

diff --git a/590.patch b/590.patch
deleted file mode 100644
index cf8d7de..0000000
--- a/590.patch
+++ /dev/null
@@ -1,23 +0,0 @@
-From ef30469e3310e2b57ba29098496fe90ca8d0f558 Mon Sep 17 00:00:00 2001
-From: Dave Dykstra <2129743+DrDaveD@users.noreply.github.com>
-Date: Tue, 2 Aug 2022 11:55:17 -0500
-Subject: [PATCH] fix 32bit compilation
-
-Signed-off-by: Dave Dykstra <2129743+DrDaveD@users.noreply.github.com>
----
- internal/pkg/util/fs/overlay/overlay_linux.go | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/internal/pkg/util/fs/overlay/overlay_linux.go b/internal/pkg/util/fs/overlay/overlay_linux.go
-index b5eff4bd2..7b220f97f 100644
---- a/internal/pkg/util/fs/overlay/overlay_linux.go
-+++ b/internal/pkg/util/fs/overlay/overlay_linux.go
-@@ -81,7 +81,7 @@ func check(path string, d dir, allowType int64) error {
- 		return nil
- 	}
- 
--	if stfs.Type == allowType {
-+	if int64(stfs.Type) == allowType {
- 		return nil
- 	}
- 

diff --git a/70.patch b/70.patch
new file mode 100644
index 0000000..9d8660d
--- /dev/null
+++ b/70.patch
@@ -0,0 +1,1784 @@
+From 2ff837b814e84eedc25574014da9ed24ee44b7b4 Mon Sep 17 00:00:00 2001
+From: Kevin Vigor <kvigor@gmail.com>
+Date: Thu, 6 Feb 2020 09:59:56 -0800
+Subject: [PATCH 1/3] Add locking to cache API in anticipation of multithreaded
+ decompression.
+
+The existing cache API has effectively no locking and relies on the single-
+threaded nature of the code to prevent contention on cache entries.
+As a first step to multithreading the driver, refactor the cache API to allow
+alternative implementations. In this changeset the API is changed but the
+internal cache implementation is not make thread-safe, so it still suitable
+only for single-threaded usage.
+
+Specific changes:
+
+*   sqfs_cache type is made opaque.
+*   previously, newly allocated cache entries were assumed valid. This meant that
+    on any failure path following a cache entry allocation, one had to be careful
+    to call sqfs_cache_invalidate(). The assumption is now reversed, cache
+    entries are invalid until explicitly marked valid with
+    sqfs_cache_entry_valid(). This simplifies error handling.
+*   block cache code was intermixed with generic cache code, relocate to fs.c
+*   cache eviction led to object destruction in block cache. The only thing
+    preventing cache eviction while block object in use is single-threaded
+    nature of code. Instead use a refcounting mechanism on the block entries
+    so that we can independently manage block lifetime.
+---
+ Makefile.am       |   6 +-
+ cache.c           | 142 +++++++++++++++++++++++++++++-----------------
+ cache.h           |  40 ++++++-------
+ common.h          |  26 ++++++++-
+ file.c            |  23 ++++----
+ file.h            |   4 --
+ fs.c              |  61 ++++++++++++++++----
+ fs.h              |   5 ++
+ table.c           |   2 +-
+ tests/cachetest.c | 107 ++++++++++++++++++++++++++++++++++
+ 10 files changed, 314 insertions(+), 102 deletions(-)
+ create mode 100644 tests/cachetest.c
+
+diff --git a/Makefile.am b/Makefile.am
+index 5659cd22..eaf7ac97 100644
+--- a/Makefile.am
++++ b/Makefile.am
+@@ -105,9 +105,11 @@ endif
+ TESTS =
+ if SQ_FUSE_TESTS
+ TESTS += tests/ll-smoke.sh
+-check_PROGRAMS = endiantest
++check_PROGRAMS = cachetest endiantest
++cachetest_SOURCES=tests/cachetest.c
++cachetest_LDADD=libsquashfuse.la $(COMPRESSION_LIBS)
+ endiantest_SOURCES = tests/endiantest.c
+-TESTS += endiantest
++TESTS += cachetest endiantest
+ endif
+ if SQ_DEMO_TESTS
+ TESTS += tests/ls.sh
+diff --git a/cache.c b/cache.c
+index 0deacfca..36d02234 100644
+--- a/cache.c
++++ b/cache.c
+@@ -22,85 +22,121 @@
+  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+  */
++
++#include "config.h"
+ #include "cache.h"
+ 
+ #include "fs.h"
+ 
++#include <assert.h>
+ #include <stdlib.h>
+ 
++typedef struct sqfs_cache_internal {
++	uint8_t *buf;
++
++	sqfs_cache_dispose dispose;
++
++	size_t size, count;
++	size_t next; /* next block to evict */
++} sqfs_cache_internal;
++
++typedef struct {
++	int valid;
++	sqfs_cache_idx idx;
++} sqfs_cache_entry_hdr;
++
+ sqfs_err sqfs_cache_init(sqfs_cache *cache, size_t size, size_t count,
+-		sqfs_cache_dispose dispose) {
+-	cache->size = size;
+-	cache->count = count;
+-	cache->dispose = dispose;
+-	cache->next = 0;
+-	
+-	cache->idxs = calloc(count, sizeof(sqfs_cache_idx));
+-	cache->buf = calloc(count, size);
+-	if (cache->idxs && cache->buf)
++			 sqfs_cache_dispose dispose) {
++
++	sqfs_cache_internal *c = malloc(sizeof(sqfs_cache_internal));
++	if (!c) {
++		return SQFS_ERR;
++	}
++
++	c->size = size + sizeof(sqfs_cache_entry_hdr);
++	c->count = count;
++	c->dispose = dispose;
++	c->next = 0;
++
++	c->buf = calloc(count, c->size);
++
++	if (c->buf) {
++		*cache = c;
+ 		return SQFS_OK;
+-	
+-	sqfs_cache_destroy(cache);
++	}
++
++	sqfs_cache_destroy(&c);
+ 	return SQFS_ERR;
+ }
+ 
+-static void *sqfs_cache_entry(sqfs_cache *cache, size_t i) {
+-	return cache->buf + i * cache->size;
++static sqfs_cache_entry_hdr *sqfs_cache_entry_header(
++						     sqfs_cache_internal* cache,
++						     size_t i) {
++	return (sqfs_cache_entry_hdr *)(cache->buf + i * cache->size);
++}
++
++static void* sqfs_cache_entry(sqfs_cache_internal* cache, size_t i) {
++	return (void *)(sqfs_cache_entry_header(cache, i) + 1);
+ }
+ 
+ void sqfs_cache_destroy(sqfs_cache *cache) {
+-	if (cache->buf && cache->idxs) {
+-		size_t i;
+-		for (i = 0; i < cache->count; ++i) {
+-			if (cache->idxs[i] != SQFS_CACHE_IDX_INVALID)
+-				cache->dispose(sqfs_cache_entry(cache, i));
++	if (cache && *cache) {
++		sqfs_cache_internal *c = *cache;
++		if (c->buf) {
++			size_t i;
++			for (i = 0; i < c->count; ++i) {
++				sqfs_cache_entry_hdr *hdr =
++					sqfs_cache_entry_header(c, i);
++				if (hdr->valid) {
++					c->dispose((void *)(hdr + 1));
++				}
++			}
+ 		}
++		free(c->buf);
++		free(c);
++		*cache = NULL;
+ 	}
+-	free(cache->buf);
+-	free(cache->idxs);
+ }
+ 
+ void *sqfs_cache_get(sqfs_cache *cache, sqfs_cache_idx idx) {
+ 	size_t i;
+-	for (i = 0; i < cache->count; ++i) {
+-		if (cache->idxs[i] == idx)
+-			return sqfs_cache_entry(cache, i);
++	sqfs_cache_internal *c = *cache;
++	sqfs_cache_entry_hdr *hdr;
++
++	for (i = 0; i < c->count; ++i) {
++		hdr = sqfs_cache_entry_header(c, i);
++		if (hdr->idx == idx) {
++			assert(hdr->valid);
++			return sqfs_cache_entry(c, i);
++		}
+ 	}
+-	return NULL;
+-}
+ 
+-void *sqfs_cache_add(sqfs_cache *cache, sqfs_cache_idx idx) {
+-	size_t i = (cache->next++);
+-	cache->next %= cache->count;
+-	
+-	if (cache->idxs[i] != SQFS_CACHE_IDX_INVALID)
+-		cache->dispose(sqfs_cache_entry(cache, i));
+-	
+-	cache->idxs[i] = idx;
+-	return sqfs_cache_entry(cache, i);
+-}
++	/* No existing entry; free one if necessary, allocate a new one. */
++	i = (c->next++);
++	c->next %= c->count;
+ 
+-/* sqfs_cache_add can be called but the caller can fail to fill it (IO
+- * error, etc).  sqfs_cache_invalidate invalidates the cache entry.
+- * It does not call dispose; it merely marks the entry as reusable
+- * since it is never fully initialized.
+- */
+-void sqfs_cache_invalidate(sqfs_cache *cache, sqfs_cache_idx idx) {
+-	size_t i;
+-	for (i = 0; i < cache->count; ++i) {
+-		if (cache->idxs[i] == idx) {
+-			cache->idxs[i] = SQFS_CACHE_IDX_INVALID;
+-			return;
+-		}
++	hdr = sqfs_cache_entry_header(c, i);
++	if (hdr->valid) {
++		/* evict */
++		c->dispose((void *)(hdr + 1));
++		hdr->valid = 0;
+ 	}
++
++	hdr->idx = idx;
++	return (void *)(hdr + 1);
++}
++
++int sqfs_cache_entry_valid(const sqfs_cache *cache, const void *e) {
++	sqfs_cache_entry_hdr *hdr = ((sqfs_cache_entry_hdr *)e) - 1;
++	return hdr->valid;
+ }
+ 
+-static void sqfs_block_cache_dispose(void *data) {
+-	sqfs_block_cache_entry *entry = (sqfs_block_cache_entry*)data;
+-	sqfs_block_dispose(entry->block);
++void sqfs_cache_entry_mark_valid(sqfs_cache *cache, void *e) {
++	sqfs_cache_entry_hdr *hdr = ((sqfs_cache_entry_hdr *)e) - 1;
++	assert(hdr->valid == 0);
++	hdr->valid = 1;
+ }
+ 
+-sqfs_err sqfs_block_cache_init(sqfs_cache *cache, size_t count) {
+-	return sqfs_cache_init(cache, sizeof(sqfs_block_cache_entry), count,
+-		&sqfs_block_cache_dispose);
++void sqfs_cache_put(const sqfs_cache *cache, const void *e) {
++	// nada, we have no locking in single-threaded implementation.
+ }
+diff --git a/cache.h b/cache.h
+index b78c524d..da471352 100644
+--- a/cache.h
++++ b/cache.h
+@@ -33,35 +33,37 @@
+  *  - No thread safety
+  *  - Misses are caller's responsibility
+  */
+-#define SQFS_CACHE_IDX_INVALID 0
+ 
+ typedef uint64_t sqfs_cache_idx;
+ typedef void (*sqfs_cache_dispose)(void* data);
+ 
+-typedef struct {
+-	sqfs_cache_idx *idxs;
+-	uint8_t *buf;
+-	
+-	sqfs_cache_dispose dispose;
+-	
+-	size_t size, count;
+-	size_t next; /* next block to evict */
+-} sqfs_cache;
++struct sqfs_cache_internal;
++typedef struct sqfs_cache_internal *sqfs_cache;
+ 
+ sqfs_err sqfs_cache_init(sqfs_cache *cache, size_t size, size_t count,
+ 	sqfs_cache_dispose dispose);
+ void sqfs_cache_destroy(sqfs_cache *cache);
+ 
++/* Get an entry for the given index.
++ *
++ * This will always succeed (evicting if necessary). The caller must then
++ * call sqfs_cache_entry_valid() to determine if the entry is valid. If not
++ * valid, the entry is newly allocated and the caller is responsible for
++ * initializing it and then calling sqfs_cache_entry_mark_valid().
++ *
++ * This call may block in multithreaded case.
++ *
++ * In multithreaded case, the cache is locked on return (no entries can
++ * be added or removed). Caller must call sqfs_cache_put() when it is safe
++ * to evict the returned cache entry.
++ */
+ void *sqfs_cache_get(sqfs_cache *cache, sqfs_cache_idx idx);
+-void *sqfs_cache_add(sqfs_cache *cache, sqfs_cache_idx idx);
+-void sqfs_cache_invalidate(sqfs_cache *cache, sqfs_cache_idx idx);
+-
+-
+-typedef struct {
+-	sqfs_block *block;
+-	size_t data_size;
+-} sqfs_block_cache_entry;
++/* inform cache it is now safe to evict this entry. */
++void sqfs_cache_put(const sqfs_cache *cache, const void *e);
+ 
+-sqfs_err sqfs_block_cache_init(sqfs_cache *cache, size_t count);
++/* Determine if cache entry contains valid contents. */
++int sqfs_cache_entry_valid(const sqfs_cache *cache, const void *e);
++/* Mark cache entry as containing valid contents. */
++void sqfs_cache_entry_mark_valid(sqfs_cache *cache, void *e);
+ 
+ #endif
+diff --git a/common.h b/common.h
+index aeac5c67..9d50e006 100644
+--- a/common.h
++++ b/common.h
+@@ -32,12 +32,23 @@
+ #include <sys/types.h>
+ 
+ #ifdef _WIN32
+-	#include <win32.h>
++# include <win32.h>
++# include <intrin.h>
++# define atomic_inc_relaxed(ptr) \
++	_InterlockedIncrement(ptr)
++# define atomic_dec_acqrel(ptr) \
++	_InterlockedDecrement(ptr)
+ #else
+ 	typedef mode_t sqfs_mode_t;
+ 	typedef uid_t sqfs_id_t;
+ 	typedef off_t sqfs_off_t;
+ 	typedef int sqfs_fd_t;
++
++# define atomic_inc_relaxed(ptr) \
++	__atomic_add_fetch(&block->refcount, 1, __ATOMIC_RELAXED)
++# define atomic_dec_acqrel(ptr) \
++	__atomic_sub_fetch(&block->refcount, 1, __ATOMIC_ACQ_REL)
++
+ #endif
+ 
+ typedef enum {
+@@ -59,6 +70,7 @@ typedef struct sqfs_inode sqfs_inode;
+ typedef struct {
+ 	size_t size;
+ 	void *data;
++	long refcount;
+ } sqfs_block;
+ 
+ typedef struct {
+@@ -66,4 +78,16 @@ typedef struct {
+ 	size_t offset;
+ } sqfs_md_cursor;
+ 
++/* Increment the refcount on the block. */
++static inline void sqfs_block_ref(sqfs_block *block) {
++	atomic_inc_relaxed(&block->refcount);
++}
++
++/* decrement the refcount on the block, return non-zero if we held the last
++ * reference.
++ */
++static inline int sqfs_block_deref(sqfs_block *block) {
++	return atomic_dec_acqrel(&block->refcount) == 0;
++}
++
+ #endif
+diff --git a/file.c b/file.c
+index a4d894eb..d09f2a7d 100644
+--- a/file.c
++++ b/file.c
+@@ -177,7 +177,7 @@ sqfs_err sqfs_read_range(sqfs *fs, sqfs_inode *inode, sqfs_off_t start,
+ 			take = (size_t)(*size);
+ 		if (block) {
+ 			memcpy(buf, (char*)block->data + data_off + read_off, take);
+-			/* BLOCK CACHED, DON'T DISPOSE */
++			sqfs_block_dispose(block);
+ 		} else {
+ 			memset(buf, 0, take);
+ 		}
+@@ -226,7 +226,7 @@ sqfs_err sqfs_blockidx_init(sqfs_cache *cache) {
+ }
+ 
+ sqfs_err sqfs_blockidx_add(sqfs *fs, sqfs_inode *inode,
+-		sqfs_blockidx_entry **out) {
++		sqfs_blockidx_entry **out, sqfs_blockidx_entry **cachep) {
+ 	size_t blocks;	/* Number of blocks in the file */
+ 	size_t md_size; /* Amount of metadata necessary to hold the blocksizes */
+ 	size_t count; 	/* Number of block-index entries necessary */
+@@ -234,10 +234,6 @@ sqfs_err sqfs_blockidx_add(sqfs *fs, sqfs_inode *inode,
+ 	sqfs_blockidx_entry *blockidx;
+ 	sqfs_blocklist bl;
+ 	
+-	/* For the cache */
+-	sqfs_cache_idx idx;
+-	sqfs_blockidx_entry **cachep;
+-
+ 	size_t i = 0;
+ 	bool first = true;
+ 	
+@@ -270,8 +266,6 @@ sqfs_err sqfs_blockidx_add(sqfs *fs, sqfs_inode *inode,
+ 		}
+ 	}
+ 
+-	idx = inode->base.inode_number + 1; /* zero means invalid */
+-	cachep = sqfs_cache_add(&fs->blockidx, idx);
+ 	*out = *cachep = blockidx;
+ 	return SQFS_OK;
+ }
+@@ -299,12 +293,16 @@ sqfs_err sqfs_blockidx_blocklist(sqfs *fs, sqfs_inode *inode,
+ 	
+ 	/* Get the index, creating it if necessary */
+ 	idx = inode->base.inode_number + 1; /* zero means invalid index */
+-	if ((bp = sqfs_cache_get(&fs->blockidx, idx))) {
++	bp = sqfs_cache_get(&fs->blockidx, idx);
++	if (sqfs_cache_entry_valid(&fs->blockidx, bp)) {
+ 		blockidx = *bp;
+ 	} else {
+-		sqfs_err err = sqfs_blockidx_add(fs, inode, &blockidx);
+-		if (err)
++		sqfs_err err = sqfs_blockidx_add(fs, inode, &blockidx, bp);
++		if (err) {
++			sqfs_cache_put(&fs->blockidx, bp);
+ 			return err;
++		}
++		sqfs_cache_entry_mark_valid(&fs->blockidx, bp);
+ 	}
+ 	
+ 	skipped = (metablock * SQUASHFS_METADATA_SIZE / sizeof(sqfs_blocklist_entry))
+@@ -316,6 +314,9 @@ sqfs_err sqfs_blockidx_blocklist(sqfs *fs, sqfs_inode *inode,
+ 	bl->remain -= skipped;
+ 	bl->pos = (uint64_t)skipped * fs->sb.block_size;
+ 	bl->block = blockidx->data_block;
++
++	sqfs_cache_put(&fs->blockidx, bp);
++
+ 	return SQFS_OK;
+ }
+ 
+diff --git a/file.h b/file.h
+index 249c6413..e3d2b028 100644
+--- a/file.h
++++ b/file.h
+@@ -71,10 +71,6 @@ typedef struct {
+ 
+ sqfs_err sqfs_blockidx_init(sqfs_cache *cache);
+ 
+-/* Fill *out with all the block-index entries for this file */
+-sqfs_err sqfs_blockidx_add(sqfs *fs, sqfs_inode *inode,
+-	sqfs_blockidx_entry **out);
+-
+ /* Get a blocklist fast-forwarded to the correct location */
+ sqfs_err sqfs_blockidx_blocklist(sqfs *fs, sqfs_inode *inode,
+ 	sqfs_blocklist *bl, sqfs_off_t start);
+diff --git a/fs.c b/fs.c
+index d69bb681..1838c5ca 100644
+--- a/fs.c
++++ b/fs.c
+@@ -124,6 +124,8 @@ sqfs_err sqfs_block_read(sqfs *fs, sqfs_off_t pos, bool compressed,
+ 	sqfs_err err = SQFS_ERR;
+ 	if (!(*block = malloc(sizeof(**block))))
+ 		return SQFS_ERR;
++	/* start with refcount one, so dispose on failure path works as expected. */
++	(*block)->refcount = 1;
+ 	if (!((*block)->data = malloc(size)))
+ 		goto error;
+ 	
+@@ -188,44 +190,81 @@ sqfs_err sqfs_data_block_read(sqfs *fs, sqfs_off_t pos, uint32_t hdr,
+ }
+ 
+ sqfs_err sqfs_md_cache(sqfs *fs, sqfs_off_t *pos, sqfs_block **block) {
+-	sqfs_block_cache_entry *entry = sqfs_cache_get(
+-		&fs->md_cache, *pos);
+-	if (!entry) {
++	sqfs_block_cache_entry *entry = sqfs_cache_get(&fs->md_cache, *pos);
++	if (!sqfs_cache_entry_valid(&fs->md_cache, entry)) {
+ 		sqfs_err err = SQFS_OK;
+-		entry = sqfs_cache_add(&fs->md_cache, *pos);
+ 		/* fprintf(stderr, "MD BLOCK: %12llx\n", (long long)*pos); */
+ 		err = sqfs_md_block_read(fs, *pos,
+ 			&entry->data_size, &entry->block);
+ 		if (err) {
+-			sqfs_cache_invalidate(&fs->md_cache, *pos);
++			sqfs_cache_put(&fs->md_cache, entry);
+ 			return err;
+ 		}
++		sqfs_cache_entry_mark_valid(&fs->md_cache, entry);
+ 	}
++	/* block is created with refcount 1, which accounts for presence in the
++	 * cache (will be decremented on eviction).
++	 *
++	 * We increment it here as a convienience for the caller, who will
++	 * obviously want one. Therefore all callers must eventually call deref
++	 * by means of calling sqfs_block_dispose().
++	 */
+ 	*block = entry->block;
+ 	*pos += entry->data_size;
++
++	sqfs_block_ref(entry->block);
++    /* it is now safe to evict the entry from the cache, we have a
++     * reference to the block so eviction will not destroy it.
++     */
++	sqfs_cache_put(&fs->md_cache, entry);
++
+ 	return SQFS_OK;
+ }
+ 
+ sqfs_err sqfs_data_cache(sqfs *fs, sqfs_cache *cache, sqfs_off_t pos,
+ 		uint32_t hdr, sqfs_block **block) {
+ 	sqfs_block_cache_entry *entry = sqfs_cache_get(cache, pos);
+-	if (!entry) {
++	if (!sqfs_cache_entry_valid(cache, entry)) {
+ 		sqfs_err err = SQFS_OK;
+-		entry = sqfs_cache_add(cache, pos);
+ 		err = sqfs_data_block_read(fs, pos, hdr,
+ 			&entry->block);
+ 		if (err) {
+-			sqfs_cache_invalidate(cache, pos);
++            sqfs_cache_put(cache, entry);
+ 			return err;
+ 		}
++		sqfs_cache_entry_mark_valid(cache, entry);
+ 	}
++	/* block is created with refcount 1, which accounts for presence in the
++	 * cache (will be decremented on eviction).
++	 *
++	 * We increment it here as a convenience for the caller, who will
++	 * obviously want one. Therefore all callers must eventually call deref
++	 * by means of calling sqfs_block_dispose().
++	 */
+ 	*block = entry->block;
++	sqfs_block_ref(*block);
++    /* it is now safe to evict the entry from the cache, we have a
++     * reference to the block so eviction will not destroy it.
++     */
++	sqfs_cache_put(cache, entry);
+ 	return SQFS_OK;
+ }
+ 
+ void sqfs_block_dispose(sqfs_block *block) {
+-	free(block->data);
+-	free(block);
++	if (sqfs_block_deref(block)) {
++		free(block->data);
++		free(block);
++	}
++}
++
++static void sqfs_block_cache_dispose(void *data) {
++	sqfs_block_cache_entry *entry = (sqfs_block_cache_entry*)data;
++	sqfs_block_dispose(entry->block);
++}
++
++sqfs_err sqfs_block_cache_init(sqfs_cache *cache, size_t count) {
++	return sqfs_cache_init(cache, sizeof(sqfs_block_cache_entry), count,
++			       &sqfs_block_cache_dispose);
+ }
+ 
+ void sqfs_md_cursor_inode(sqfs_md_cursor *cur, sqfs_inode_id id, sqfs_off_t base) {
+@@ -247,7 +286,6 @@ sqfs_err sqfs_md_read(sqfs *fs, sqfs_md_cursor *cur, void *buf, size_t size) {
+ 			take = size;		
+ 		if (buf)
+ 			memcpy(buf, (char*)block->data + cur->offset, take);
+-		/* BLOCK CACHED, DON'T DISPOSE */
+ 		
+ 		if (buf)
+ 			buf = (char*)buf + take;
+@@ -257,6 +295,7 @@ sqfs_err sqfs_md_read(sqfs *fs, sqfs_md_cursor *cur, void *buf, size_t size) {
+ 			cur->block = pos;
+ 			cur->offset = 0;
+ 		}
++		sqfs_block_dispose(block);
+ 	}
+ 	return SQFS_OK;
+ }
+diff --git a/fs.h b/fs.h
+index d300a3bb..1d475ce0 100644
+--- a/fs.h
++++ b/fs.h
+@@ -97,6 +97,11 @@ sqfs_compression_type sqfs_compression(sqfs *fs);
+ void sqfs_md_header(uint16_t hdr, bool *compressed, uint16_t *size);
+ void sqfs_data_header(uint32_t hdr, bool *compressed, uint32_t *size);
+ 
++typedef struct {
++	sqfs_block *block;
++	size_t data_size;
++} sqfs_block_cache_entry;
++sqfs_err sqfs_block_cache_init(sqfs_cache *cache, size_t count);
+ sqfs_err sqfs_block_read(sqfs *fs, sqfs_off_t pos, bool compressed, uint32_t size,
+ 	size_t outsize, sqfs_block **block);
+ void sqfs_block_dispose(sqfs_block *block);
+diff --git a/table.c b/table.c
+index c035398f..02a5442c 100644
+--- a/table.c
++++ b/table.c
+@@ -76,6 +76,6 @@ sqfs_err sqfs_table_get(sqfs_table *table, sqfs *fs, size_t idx, void *buf) {
+ 		return SQFS_ERR;
+ 	
+ 	memcpy(buf, (char*)(block->data) + off, table->each);
+-	/* BLOCK CACHED, DON'T DISPOSE */
++	sqfs_block_dispose(block);
+ 	return SQFS_OK;
+ }
+diff --git a/tests/cachetest.c b/tests/cachetest.c
+new file mode 100644
+index 00000000..8a2c2363
+--- /dev/null
++++ b/tests/cachetest.c
+@@ -0,0 +1,107 @@
++#include "cache.h"
++#include <stdio.h>
++
++typedef struct {
++    int x;
++    int y;
++} TestStruct;
++
++static void TestStructDispose(void *t) {
++    // nada.
++}
++
++#define EXPECT_EQ(exp1, exp2)						  \
++	do { if ((exp1) != (exp2)) {					  \
++		printf("Test failure: expected " #exp1 " to equal " #exp2 \
++		       " at " __FILE__ ":%d\n", __LINE__);		  \
++		++errors;						  \
++	  }								  \
++	} while (0)
++
++#define EXPECT_NE(exp1, exp2)						   \
++	do { if ((exp1) == (exp2)) {					   \
++		printf("Test failure: expected " #exp1 " to !equal " #exp2 \
++		       " at " __FILE__ ":%d\n", __LINE__);		   \
++		++errors;						   \
++          }								   \
++	} while (0)
++
++
++int test_cache_miss(void) {
++    int errors = 0;
++    sqfs_cache cache;
++    TestStruct *entry;
++
++    EXPECT_EQ(sqfs_cache_init(&cache, sizeof(TestStruct), 16,
++                              TestStructDispose), SQFS_OK);
++    entry = (TestStruct *)sqfs_cache_get(&cache, 1);
++    EXPECT_EQ(sqfs_cache_entry_valid(&cache, entry), 0);
++    sqfs_cache_destroy(&cache);
++
++    return errors == 0;
++}
++
++int test_mark_valid_and_lookup(void) {
++    int errors = 0;
++    sqfs_cache cache;
++    TestStruct *entry;
++
++    EXPECT_EQ(sqfs_cache_init(&cache, sizeof(TestStruct), 16,
++                              TestStructDispose), SQFS_OK);
++    entry = (TestStruct *)sqfs_cache_get(&cache, 1);
++    entry->x = 666;
++    entry->y = 777;
++    sqfs_cache_entry_mark_valid(&cache, entry);
++    sqfs_cache_put(&cache, entry);
++    EXPECT_NE(sqfs_cache_entry_valid(&cache, entry), 0);
++    entry = (TestStruct *)sqfs_cache_get(&cache, 1);
++    EXPECT_NE(sqfs_cache_entry_valid(&cache, entry), 0);
++    EXPECT_EQ(entry->x, 666);
++    EXPECT_EQ(entry->y, 777);
++    sqfs_cache_put(&cache, entry);
++
++    sqfs_cache_destroy(&cache);
++    return errors == 0;
++}
++
++int test_two_entries(void) {
++    int errors = 0;
++    sqfs_cache cache;
++    TestStruct *entry1, *entry2;
++
++    EXPECT_EQ(sqfs_cache_init(&cache, sizeof(TestStruct), 16,
++                              TestStructDispose), SQFS_OK);
++
++    entry1 = (TestStruct *)sqfs_cache_get(&cache, 1);
++    entry1->x = 1;
++    entry1->y = 2;
++    sqfs_cache_entry_mark_valid(&cache, entry1);
++    sqfs_cache_put(&cache, entry1);
++
++    entry2 = (TestStruct *)sqfs_cache_get(&cache, 666);
++    entry2->x = 3;
++    entry2->y = 4;
++    sqfs_cache_entry_mark_valid(&cache, entry2);
++    sqfs_cache_put(&cache, entry2);
++
++    entry1 = (TestStruct *)sqfs_cache_get(&cache, 1);
++    sqfs_cache_put(&cache, entry1);
++    entry2 = (TestStruct *)sqfs_cache_get(&cache, 666);
++    sqfs_cache_put(&cache, entry2);
++    EXPECT_NE(sqfs_cache_entry_valid(&cache, entry1), 0);
++    EXPECT_NE(sqfs_cache_entry_valid(&cache, entry2), 0);
++    EXPECT_EQ(entry1->x, 1);
++    EXPECT_EQ(entry1->y, 2);
++    EXPECT_EQ(entry2->x, 3);
++    EXPECT_EQ(entry2->y, 4);
++
++    sqfs_cache_destroy(&cache);
++
++    return errors == 0;
++}
++
++int main(void) {
++	return test_cache_miss() &&
++		test_mark_valid_and_lookup() &&
++		test_two_entries() ? 0 : 1;
++}
+
+From 379c8507c15ef43b641c1024b43372f2de9fb480 Mon Sep 17 00:00:00 2001
+From: Kevin Vigor <kvigor@gmail.com>
+Date: Thu, 6 Feb 2020 12:20:19 -0800
+Subject: [PATCH 2/3] Implement multi-threaded squashfuse_ll, allowing parallel
+ decompression.
+
+A simple thread-safe cache implementation is added and squashfuse_ll init
+is altered to use fuse_session_loop_mt().
+
+Multithreading must be explicitly enabled at configure time with the
+--enable-multithreading option. If enabled, the resulting squashfuse_ll
+will be multithreaded by default, but this may be disabled at runtime
+with the '-s' FUSE commandline option.
+---
+ Makefile.am                      |   8 +-
+ cache.c                          |   4 +
+ cache_mt.c                       | 169 +++++++++++++++++++++++++++++++
+ configure.ac                     |  11 +-
+ fs.c                             |   9 +-
+ ll.c                             |  77 ++++++++++----
+ ll_main.c                        |  20 +++-
+ m4/squashfuse_c.m4               |  33 +-----
+ squashfs_fs.h                    |   6 +-
+ tests/cachetest.c                |   1 +
+ tests/ll-smoke-singlethreaded.sh |  10 ++
+ tests/ll-smoke.sh                | 141 ++++++++++++++++++++++++++
+ tests/ll-smoke.sh.in             |   6 +-
+ 13 files changed, 432 insertions(+), 63 deletions(-)
+ create mode 100644 cache_mt.c
+ create mode 100755 tests/ll-smoke-singlethreaded.sh
+ create mode 100755 tests/ll-smoke.sh
+
+diff --git a/Makefile.am b/Makefile.am
+index eaf7ac97..17b01be4 100644
+--- a/Makefile.am
++++ b/Makefile.am
+@@ -26,7 +26,7 @@ pkgconfig_DATA 	= squashfuse.pc
+ noinst_LTLIBRARIES += libsquashfuse_convenience.la
+ libsquashfuse_convenience_la_SOURCES = swap.c cache.c table.c dir.c file.c fs.c \
+ 	decompress.c xattr.c hash.c stack.c traverse.c util.c \
+-	nonstd-pread.c nonstd-stat.c \
++	nonstd-pread.c nonstd-stat.c cache_mt.c \
+ 	squashfs_fs.h common.h nonstd-internal.h nonstd.h swap.h cache.h table.h \
+ 	dir.h file.h decompress.h xattr.h squashfuse.h hash.h stack.h traverse.h \
+ 	util.h fs.h
+@@ -105,6 +105,12 @@ endif
+ TESTS =
+ if SQ_FUSE_TESTS
+ TESTS += tests/ll-smoke.sh
++if MULTITHREADED
++# I know this test looks backwards, but the default smoke test is multithreaded
++# when threading is enabled. So we additionally run a singlethreaded test in
++# that case.
++TESTS += tests/ll-smoke-singlethreaded.sh
++endif
+ check_PROGRAMS = cachetest endiantest
+ cachetest_SOURCES=tests/cachetest.c
+ cachetest_LDADD=libsquashfuse.la $(COMPRESSION_LIBS)
+diff --git a/cache.c b/cache.c
+index 36d02234..45408f24 100644
+--- a/cache.c
++++ b/cache.c
+@@ -24,6 +24,9 @@
+  */
+ 
+ #include "config.h"
++
++#ifndef SQFS_MULTITHREADED
++
+ #include "cache.h"
+ 
+ #include "fs.h"
+@@ -140,3 +143,4 @@ void sqfs_cache_entry_mark_valid(sqfs_cache *cache, void *e) {
+ void sqfs_cache_put(const sqfs_cache *cache, const void *e) {
+ 	// nada, we have no locking in single-threaded implementation.
+ }
++#endif /* SQFS_MULTITHREADED */
+diff --git a/cache_mt.c b/cache_mt.c
+new file mode 100644
+index 00000000..1b17fa5a
+--- /dev/null
++++ b/cache_mt.c
+@@ -0,0 +1,169 @@
++#include "config.h"
++
++#ifdef SQFS_MULTITHREADED
++
++/* Thread-safe cache implementation.
++ *
++ * Simple implementation: basic hash table, each individual entry is
++ * protected by a mutex, any collision is handled by eviction.
++ */
++
++#include "cache.h"
++#include "fs.h"
++
++#include <assert.h>
++#include <pthread.h>
++#include <stdlib.h>
++
++typedef struct sqfs_cache_internal {
++    uint8_t *buf;
++    sqfs_cache_dispose dispose;
++    size_t entry_size, count;
++} sqfs_cache_internal;
++
++typedef struct {
++    enum { EMPTY, FULL } state;
++    sqfs_cache_idx idx;
++    pthread_mutex_t lock;
++} sqfs_cache_entry_hdr;
++
++// MurmurHash64A performance-optimized for hash of uint64_t keys
++const static uint64_t kMurmur2Seed = 4193360111ul;
++static uint64_t MurmurRehash64A(uint64_t key) {
++    const uint64_t m = 0xc6a4a7935bd1e995;
++    const int r = 47;
++
++    uint64_t h = (uint64_t)kMurmur2Seed ^ (sizeof(uint64_t) * m);
++
++    key *= m;
++    key ^= key >> r;
++    key *= m;
++
++    h ^= key;
++    h *= m;
++
++    h ^= h >> r;
++    h *= m;
++    h ^= h >> r;
++
++    return h;
++}
++
++static sqfs_cache_entry_hdr *sqfs_cache_entry_header(
++                             sqfs_cache_internal* cache,
++                             size_t i) {
++    assert(i < cache->count);
++    return (sqfs_cache_entry_hdr *)(cache->buf + i * cache->entry_size);
++}
++
++sqfs_err sqfs_cache_init(sqfs_cache *cache, size_t entry_size, size_t count,
++             sqfs_cache_dispose dispose) {
++    size_t i;
++    pthread_mutexattr_t attr;
++    sqfs_cache_internal *c = malloc(sizeof(sqfs_cache_internal));
++
++    if (!c) {
++        return SQFS_ERR;
++    }
++
++    c->entry_size = entry_size + sizeof(sqfs_cache_entry_hdr);
++    c->count = count;
++    c->dispose = dispose;
++
++    pthread_mutexattr_init(&attr);
++#if defined(_GNU_SOURCE) && !defined(NDEBUG)
++    pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK);
++#endif
++
++    c->buf = calloc(c->count, c->entry_size);
++    if (!c->buf) {
++        goto err_out;
++    }
++
++    for (i = 0; i < c->count; ++i) {
++        sqfs_cache_entry_hdr *hdr = sqfs_cache_entry_header(c, i);
++        hdr->state = EMPTY;
++        if (pthread_mutex_init(&hdr->lock, &attr)) {
++            goto err_out;
++        }
++    }
++
++    pthread_mutexattr_destroy(&attr);
++
++    *cache = c;
++    return SQFS_OK;
++
++err_out:
++    sqfs_cache_destroy(&c);
++    return SQFS_ERR;
++}
++
++void sqfs_cache_destroy(sqfs_cache *cache) {
++    if (cache && *cache) {
++        sqfs_cache_internal *c = *cache;
++        if (c->buf) {
++            size_t i;
++            for (i = 0; i < c->count; ++i) {
++                sqfs_cache_entry_hdr *hdr =
++                    sqfs_cache_entry_header(c, i);
++                if (hdr->state == FULL) {
++                    c->dispose((void *)(hdr + 1));
++                }
++                if (pthread_mutex_destroy(&hdr->lock)) {
++                    assert(0);
++                }
++            }
++        }
++        free(c->buf);
++        free(c);
++        *cache = NULL;
++    }
++}
++
++void *sqfs_cache_get(sqfs_cache *cache, sqfs_cache_idx idx) {
++    sqfs_cache_internal *c = *cache;
++    sqfs_cache_entry_hdr *hdr;
++    void *entry;
++
++    uint64_t key = MurmurRehash64A(idx) % c->count;
++
++    hdr = sqfs_cache_entry_header(c, key);
++    if (pthread_mutex_lock(&hdr->lock)) { assert(0); }
++    /* matching unlock is in sqfs_cache_put() */
++    entry = (void *)(hdr + 1);
++
++    if (hdr->state == EMPTY) {
++        hdr->idx = idx;
++        return entry;
++    }
++
++    /* There's a valid entry: it's either a cache hit or a collision. */
++    assert(hdr->state == FULL);
++    if (hdr->idx == idx) {
++        return entry;
++    }
++
++    /* Collision. */
++    c->dispose((void *)(hdr + 1));
++    hdr->state = EMPTY;
++    hdr->idx = idx;
++    return entry;
++}
++
++int sqfs_cache_entry_valid(const sqfs_cache *cache, const void *e) {
++    sqfs_cache_entry_hdr *hdr = ((sqfs_cache_entry_hdr *)e) - 1;
++    return hdr->state == FULL;
++}
++
++void sqfs_cache_entry_mark_valid(sqfs_cache *cache, void *e) {
++    sqfs_cache_entry_hdr *hdr = ((sqfs_cache_entry_hdr *)e) - 1;
++    assert(hdr->state == EMPTY);
++    hdr->state = FULL;
++}
++
++void sqfs_cache_put(const sqfs_cache *cache, const void *e) {
++    sqfs_cache_entry_hdr *hdr = ((sqfs_cache_entry_hdr *)e) - 1;
++    if (pthread_mutex_unlock(&hdr->lock)) { assert(0); }
++}
++
++#endif /* SQFS_MULTITHREADED */
+diff --git a/configure.ac b/configure.ac
+index 762766e9..3869075a 100644
+--- a/configure.ac
++++ b/configure.ac
+@@ -10,6 +10,7 @@ AH_BOTTOM([#endif])
+ AC_CANONICAL_BUILD
+ AC_CANONICAL_TARGET
+ AM_INIT_AUTOMAKE([foreign -Wall subdir-objects])
++AC_USE_SYSTEM_EXTENSIONS
+ AM_SILENT_RULES(yes)
+ AM_PROG_AR
+ LT_INIT
+@@ -23,10 +24,8 @@ AC_PROG_SED
+ AC_PROG_CPP
+ AC_SYS_LARGEFILE
+ AM_PROG_CC_C_O
+-SQ_PROG_CPP_POSIX_2001
+ SQ_PROG_CC_WALL
+ 
+-AC_DEFINE([_POSIX_C_SOURCE], [200112L], [POSIX 2001 compatibility])
+ 
+ # Non-POSIX declarations
+ SQ_CHECK_DECL_MAKEDEV
+@@ -97,6 +96,14 @@ AC_CONFIG_FILES([tests/ll-smoke.sh],[chmod +x tests/ll-smoke.sh])
+ AS_IF([test "x$sq_high_level$sq_low_level$sq_demo" = xnonono],
+ 	AC_MSG_FAILURE([Nothing left to build]))
+ 
++AC_ARG_ENABLE([multithreading],
++ 	AS_HELP_STRING([--enable-multithreading], [enable multi-threaded low-level FUSE driver]),
++	[
++    AC_CHECK_LIB([pthread], [pthread_mutex_lock], [], AC_MSG_ERROR([libpthread is required for multithreaded build]))
++    AC_DEFINE(SQFS_MULTITHREADED, 1, [Enable multi-threaded low-level FUSE driver])
++    ])
++AM_CONDITIONAL([MULTITHREADED], [test x$enable_multithreading = xyes])
++
+ AC_SUBST([sq_decompressors])
+ AC_SUBST([sq_high_level])
+ AC_SUBST([sq_low_level])
+diff --git a/fs.c b/fs.c
+index 1838c5ca..ab854b0f 100644
+--- a/fs.c
++++ b/fs.c
+@@ -34,8 +34,13 @@
+ #include <sys/stat.h>
+ 
+ 
+-#define DATA_CACHED_BLKS 1
+-#define FRAG_CACHED_BLKS 3
++#ifdef SQFS_MULTITHREADED
++# define DATA_CACHED_BLKS 48
++# define FRAG_CACHED_BLKS 48
++#else
++# define DATA_CACHED_BLKS 1
++# define FRAG_CACHED_BLKS 3
++#endif
+ 
+ void sqfs_version_supported(int *min_major, int *min_minor, int *max_major,
+ 		int *max_minor) {
+diff --git a/ll.c b/ll.c
+index 4d17ba5b..596c8bf1 100644
+--- a/ll.c
++++ b/ll.c
+@@ -52,11 +52,49 @@ static sig_atomic_t open_refcount = 0;
+ /* same as lib/fuse_signals.c */
+ static struct fuse_session *fuse_instance = NULL;
+ 
++static void update_access_time(void) {
++#ifdef SQFS_MULTITHREADED
++	/* We only need to track access time if we have an idle timeout,
++	 * don't bother with expensive operations if idle_timeout is 0.
++	 */
++	if (idle_timeout_secs) {
++		time_t now = time(NULL);
++		__atomic_store_n(&last_access, now, __ATOMIC_RELEASE);
++	}
++#else
++	last_access = time(NULL);
++#endif
++}
++
++static void update_open_refcount(int delta) {
++#ifdef SQFS_MULTITHREADED
++	__atomic_fetch_add(&open_refcount, delta, __ATOMIC_RELEASE);
++#else
++	open_refcount += delta;
++#endif
++}
++
++static inline time_t get_access_time(void) {
++#ifdef SQFS_MULTITHREADED
++	return __atomic_load_n(&last_access, __ATOMIC_ACQUIRE);
++#else
++	return last_access;
++#endif
++}
++
++static inline sig_atomic_t get_open_refcount(void) {
++#ifdef SQFS_MULTITHREADED
++	return __atomic_load_n(&open_refcount, __ATOMIC_ACQUIRE);
++#else
++	return open_refcount;
++#endif
++}
++
+ void sqfs_ll_op_getattr(fuse_req_t req, fuse_ino_t ino,
+ 		struct fuse_file_info *fi) {
+ 	sqfs_ll_i lli;
+ 	struct stat st;
+-	last_access = time(NULL);
++	update_access_time();
+ 	if (sqfs_ll_iget(req, &lli, ino))
+ 		return;
+ 	
+@@ -71,7 +109,7 @@ void sqfs_ll_op_getattr(fuse_req_t req, fuse_ino_t ino,
+ void sqfs_ll_op_opendir(fuse_req_t req, fuse_ino_t ino,
+ 		struct fuse_file_info *fi) {
+ 	sqfs_ll_i *lli;
+-	last_access = time(NULL);
++	update_access_time();
+ 	
+ 	fi->fh = (intptr_t)NULL;
+ 	
+@@ -86,7 +124,7 @@ void sqfs_ll_op_opendir(fuse_req_t req, fuse_ino_t ino,
+ 			fuse_reply_err(req, ENOTDIR);
+ 		} else {
+ 			fi->fh = (intptr_t)lli;
+-			++open_refcount;
++			update_open_refcount(1);
+ 			fuse_reply_open(req, fi);
+ 			return;
+ 		}
+@@ -96,14 +134,14 @@ void sqfs_ll_op_opendir(fuse_req_t req, fuse_ino_t ino,
+ 
+ void sqfs_ll_op_create(fuse_req_t req, fuse_ino_t parent, const char *name,
+ 			      mode_t mode, struct fuse_file_info *fi) {
+-	last_access = time(NULL);
++	update_access_time();
+ 	fuse_reply_err(req, EROFS);
+ }
+ 
+ void sqfs_ll_op_releasedir(fuse_req_t req, fuse_ino_t ino,
+ 		struct fuse_file_info *fi) {
+-	last_access = time(NULL);
+-	--open_refcount;
++	update_access_time();
++	update_open_refcount(-1);
+ 	free((sqfs_ll_i*)(intptr_t)fi->fh);
+ 	fuse_reply_err(req, 0); /* yes, this is necessary */
+ }
+@@ -132,7 +170,7 @@ void sqfs_ll_op_readdir(fuse_req_t req, fuse_ino_t ino, size_t size,
+ 	sqfs_ll_i *lli = (sqfs_ll_i*)(intptr_t)fi->fh;
+ 	int err = 0;
+ 	
+-	last_access = time(NULL);
++	update_access_time();
+ 	if (sqfs_dir_open(&lli->ll->fs, &lli->inode, &dir, off))
+ 		err = EINVAL;
+ 	if (!err && !(bufpos = buf = malloc(size)))
+@@ -173,7 +211,7 @@ void sqfs_ll_op_lookup(fuse_req_t req, fuse_ino_t parent,
+ 	bool found;
+ 	sqfs_inode inode;
+ 	
+-	last_access = time(NULL);
++	update_access_time();
+ 	if (sqfs_ll_iget(req, &lli, parent))
+ 		return;
+ 	
+@@ -223,7 +261,7 @@ void sqfs_ll_op_open(fuse_req_t req, fuse_ino_t ino,
+ 	sqfs_inode *inode;
+ 	sqfs_ll *ll;
+ 	
+-	last_access = time(NULL);
++	update_access_time();
+ 	if (fi->flags & (O_WRONLY | O_RDWR)) {
+ 		fuse_reply_err(req, EROFS);
+ 		return;
+@@ -243,7 +281,7 @@ void sqfs_ll_op_open(fuse_req_t req, fuse_ino_t ino,
+ 	} else {
+ 		fi->fh = (intptr_t)inode;
+ 		fi->keep_cache = 1;
+-		++open_refcount;
++		update_open_refcount(1);
+ 		fuse_reply_open(req, fi);
+ 		return;
+ 	}
+@@ -254,8 +292,8 @@ void sqfs_ll_op_release(fuse_req_t req, fuse_ino_t ino,
+ 		struct fuse_file_info *fi) {
+ 	free((sqfs_inode*)(intptr_t)fi->fh);
+ 	fi->fh = 0;
+-	last_access = time(NULL);
+-	--open_refcount;
++	update_access_time();
++	update_open_refcount(-1);
+ 	fuse_reply_err(req, 0);
+ }
+ 
+@@ -272,7 +310,7 @@ void sqfs_ll_op_read(fuse_req_t req, fuse_ino_t ino,
+ 		return;
+ 	}
+ 	
+-	last_access = time(NULL);
++	update_access_time();
+ 	osize = size;
+ 	err = sqfs_read_range(&ll->fs, inode, off, &osize, buf);
+ 	if (err) {
+@@ -289,7 +327,7 @@ void sqfs_ll_op_readlink(fuse_req_t req, fuse_ino_t ino) {
+ 	char *dst;
+ 	size_t size;
+ 	sqfs_ll_i lli;
+-	last_access = time(NULL);
++	update_access_time();
+ 	if (sqfs_ll_iget(req, &lli, ino))
+ 		return;
+ 	
+@@ -313,7 +351,7 @@ void sqfs_ll_op_listxattr(fuse_req_t req, fuse_ino_t ino, size_t size) {
+ 	char *buf;
+ 	int ferr;
+ 	
+-	last_access = time(NULL);
++	update_access_time();
+ 	if (sqfs_ll_iget(req, &lli, ino))
+ 		return;
+ 
+@@ -351,7 +389,7 @@ void sqfs_ll_op_getxattr(fuse_req_t req, fuse_ino_t ino,
+ 	}
+ #endif
+ 	
+-	last_access = time(NULL);
++	update_access_time();
+ 	if (sqfs_ll_iget(req, &lli, ino))
+ 		return;
+ 	
+@@ -373,7 +411,7 @@ void sqfs_ll_op_getxattr(fuse_req_t req, fuse_ino_t ino,
+ void sqfs_ll_op_forget(fuse_req_t req, fuse_ino_t ino,
+ 		unsigned long nlookup) {
+ 	sqfs_ll_i lli;
+-	last_access = time(NULL);
++	update_access_time();
+ 	sqfs_ll_iget(req, &lli, SQFS_FUSE_INODE_NONE);
+ 	lli.ll->ino_forget(lli.ll, ino, nlookup);
+ 	fuse_reply_none(req);
+@@ -489,7 +527,8 @@ void alarm_tick(int sig) {
+ 		return;
+ 	}
+ 
+-	if (open_refcount == 0 && time(NULL) - last_access > idle_timeout_secs) {
++	if (get_open_refcount() == 0 &&
++		time(NULL) - get_access_time() > idle_timeout_secs) {
+ 		/* Safely shutting down fuse in a cross-platform way is a dark art!
+ 		   But just about any platform should stop on SIGINT, so do that */
+ 		kill(getpid(), SIGINT);
+@@ -499,8 +538,8 @@ void alarm_tick(int sig) {
+ }
+ 
+ void setup_idle_timeout(struct fuse_session *se, unsigned int timeout_secs) {
+-	last_access = time(NULL);
+ 	idle_timeout_secs = timeout_secs;
++	update_access_time();
+ 
+ 	struct sigaction sa;
+ 	memset(&sa, 0, sizeof(struct sigaction));
+diff --git a/ll_main.c b/ll_main.c
+index aca76935..22302085 100644
+--- a/ll_main.c
++++ b/ll_main.c
+@@ -142,8 +142,22 @@ int main(int argc, char *argv[]) {
+ 					if (opts.idle_timeout_secs) {
+ 						setup_idle_timeout(ch.session, opts.idle_timeout_secs);
+ 					}
+-					/* FIXME: multithreading */
+-					err = fuse_session_loop(ch.session);
++#ifdef SQFS_MULTITHREADED
++# if FUSE_USE_VERSION >= 30
++                    if (!fuse_cmdline_opts.singlethread) {
++                        struct fuse_loop_config config;
++                        config.clone_fd = 1;
++                        config.max_idle_threads = 10;
++                        err = fuse_session_loop_mt(ch.session, &config);
++                    }
++# else /* FUSE_USE_VERSION < 30 */
++		    if (fuse_cmdline_opts.mt) {
++			err = fuse_session_loop_mt(ch.session);
++		    }
++# endif /* FUSE_USE_VERSION */
++                    else
++#endif
++					    err = fuse_session_loop(ch.session);
+ 					teardown_idle_timeout();
+ 					fuse_remove_signal_handlers(ch.session);
+ 				}
+@@ -157,4 +171,4 @@ int main(int argc, char *argv[]) {
+ 	free(fuse_cmdline_opts.mountpoint);
+ 	
+ 	return -err;
+-}
+\ No newline at end of file
++}
+diff --git a/m4/squashfuse_c.m4 b/m4/squashfuse_c.m4
+index f29a90b1..c4039c42 100644
+--- a/m4/squashfuse_c.m4
++++ b/m4/squashfuse_c.m4
+@@ -21,37 +21,6 @@
+ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ 
+-# SQ_PROG_CPP_POSIX_2001
+-#
+-# Check if a preprocessor flag is needed for POSIX-2001 headers.
+-# Needed at least on Solaris and derivatives.
+-AC_DEFUN([SQ_PROG_CPP_POSIX_2001],[
+-AC_CACHE_CHECK([for option for POSIX-2001 preprocessor], 
+-	[sq_cv_prog_cpp_posix2001],
+-[
+-	sq_cv_prog_cpp_posix2001=unknown
+-	sq_save_CPPFLAGS=$CPPFLAGS
+-	for sq_flags in none -std=gnu99 -xc99=all
+-	do
+-		AS_IF([test "x$sq_flags" = xnone],,
+-			[CPPFLAGS="$save_CPPFLAGS $sq_flags"])
+-		AC_PREPROC_IFELSE([AC_LANG_PROGRAM([
+-			#define _POSIX_C_SOURCE 200112L
+-			#include <sys/types.h>
+-		])],[
+-			sq_cv_prog_cpp_posix2001=$sq_flags
+-			break
+-		])
+-	done
+-	CPPFLAGS=$sq_save_CPPFLAGS
+-])
+-AS_IF([test "x$sq_cv_prog_cpp_posix2001" = xunknown],
+-	[AC_MSG_FAILURE([can't preprocess for POSIX-2001])],
+-	[AS_IF([test "x$sq_cv_prog_cpp_posix2001" = xnone],,
+-		CPPFLAGS="$CPPFLAGS $sq_cv_prog_cpp_posix2001")
+-])
+-])
+-
+ # SQ_PROG_CC_WALL
+ #
+ # Check if -Wall is supported
+@@ -67,4 +36,4 @@ AC_CACHE_CHECK([how to enable all compiler warnings],
+ ])
+ AS_IF([test "x$sq_cv_prog_cc_wall" = xunknown],,
+ 	[AC_SUBST([AM_CFLAGS],["$AM_CFLAGS $sq_cv_prog_cc_wall"])])
+-])
+\ No newline at end of file
++])
+diff --git a/squashfs_fs.h b/squashfs_fs.h
+index e0ab1f4e..a85b7606 100644
+--- a/squashfs_fs.h
++++ b/squashfs_fs.h
+@@ -105,7 +105,11 @@
+ 
+ 
+ /* cached data constants for filesystem */
+-#define SQUASHFS_CACHED_BLKS		8
++#ifdef SQFS_MULTITHREADED
++# define SQUASHFS_CACHED_BLKS		128
++#else
++# define SQUASHFS_CACHED_BLKS		8
++#endif
+ 
+ #define SQUASHFS_MAX_FILE_SIZE_LOG	64
+ 
+diff --git a/tests/cachetest.c b/tests/cachetest.c
+index 8a2c2363..c515fcdf 100644
+--- a/tests/cachetest.c
++++ b/tests/cachetest.c
+@@ -36,6 +36,7 @@ int test_cache_miss(void) {
+                               TestStructDispose), SQFS_OK);
+     entry = (TestStruct *)sqfs_cache_get(&cache, 1);
+     EXPECT_EQ(sqfs_cache_entry_valid(&cache, entry), 0);
++    sqfs_cache_put(&cache, entry);
+     sqfs_cache_destroy(&cache);
+ 
+     return errors == 0;
+diff --git a/tests/ll-smoke-singlethreaded.sh b/tests/ll-smoke-singlethreaded.sh
+new file mode 100755
+index 00000000..c7cbfc38
+--- /dev/null
++++ b/tests/ll-smoke-singlethreaded.sh
+@@ -0,0 +1,10 @@
++!/bin/bash
++
++# Singlethreaded ll-smoke test.
++#
++# When multithreading is enabled at build time, it is the default
++# behavior of squashfuse_ll, but can be disabled at runtime with
++# the FUSE '-s' commandline option.
++#
++# So we just re-run the normal ll-smoke test with the '-s' option.
++SFLL_EXTRA_ARGS="-s" $(dirname -- $0)/ll-smoke.sh
+diff --git a/tests/ll-smoke.sh b/tests/ll-smoke.sh
+new file mode 100755
+index 00000000..6b9f4641
+--- /dev/null
++++ b/tests/ll-smoke.sh
+@@ -0,0 +1,141 @@
++#!/bin/sh
++
++. "tests/lib.sh"
++
++# Very simple smoke test for squashfuse_ll. Make some random files.
++# assemble a squashfs image, mount it, compare the files.
++
++SFLL=${1:-./squashfuse_ll}         # The squashfuse_ll binary.
++
++IDLE_TIMEOUT=5
++
++trap cleanup EXIT
++set -e
++
++WORKDIR=$(mktemp -d)
++
++sq_umount() {
++    case linux-gnu in
++        linux*)
++            fusermount3 -u $1
++            ;;
++        *)
++            umount $1
++            ;;
++    esac
++}
++
++sq_is_mountpoint() {
++    mount | grep -q "$1"
++}
++
++cleanup() {
++    set +e # Don't care about errors here.
++    if [ -n "$WORKDIR" ]; then
++        if [ -n "$SQ_SAVE_LOGS" ]; then
++            cp "$WORKDIR/squashfs_ll.log" "$SQ_SAVE_LOGS" || true
++        fi
++        if sq_is_mountpoint "$WORKDIR/mount"; then
++            sq_umount "$WORKDIR/mount"
++        fi
++        rm -rf "$WORKDIR"
++    fi
++}
++
++find_compressors
++
++echo "Generating random test files..."
++mkdir -p "$WORKDIR/source"
++head -c 64000000 /dev/urandom >"$WORKDIR/source/rand1"
++head -c 17000 /dev/urandom >"$WORKDIR/source/rand2"
++head -c 100000000 /dev/urandom >"$WORKDIR/source/rand3"
++head -c 87 /dev/zero >"$WORKDIR/source/z1 with spaces"
++
++for comp in $compressors; do
++    echo "Building $comp squashfs image..."
++    mksquashfs "$WORKDIR/source" "$WORKDIR/squashfs.image" -comp $comp -no-progress
++
++    mkdir -p "$WORKDIR/mount"
++
++    echo "Mounting squashfs image..."
++    $SFLL -f $SFLL_EXTRA_ARGS "$WORKDIR/squashfs.image" "$WORKDIR/mount" >"$WORKDIR/squashfs_ll.log" 2>&1 &
++    # Wait up to 5 seconds to be mounted. TSAN builds can take some time to mount.
++    for _ in $(seq 5); do
++    if sq_is_mountpoint "$WORKDIR/mount"; then
++        break
++    fi
++    sleep 1
++    done
++
++    if ! sq_is_mountpoint "$WORKDIR/mount"; then
++        echo "Image did not mount after 5 seconds."
++        cp "$WORKDIR/squashfs_ll.log" /tmp/squashfs_ll.smoke.log
++        echo "There may be clues in /tmp/squashfs_ll.smoke.log"
++        exit 1
++    fi
++
++    if command -v fio >/dev/null; then
++        echo "FIO tests..."
++        fio --filename="$WORKDIR/mount/rand1" --direct=1 --rw=randread --ioengine=libaio --bs=512 --iodepth=16 --numjobs=4 --name=j1 --minimal --output=/dev/null --runtime 30
++        fio --filename="$WORKDIR/mount/rand2" --rw=randread --ioengine=libaio --bs=4k --iodepth=16 --numjobs=4 --name=j2 --minimal --output=/dev/null --runtime 30
++        fio --filename="$WORKDIR/mount/rand3" --rw=randread --ioengine=psync --bs=128k --name=j3 --minimal --output=/dev/null --runtime 30
++    else
++        echo "Consider installing fio for better test coverage."
++    fi
++
++    echo "Comparing files..."
++    cmp "$WORKDIR/source/rand1" "$WORKDIR/mount/rand1"
++    cmp "$WORKDIR/source/rand2" "$WORKDIR/mount/rand2"
++    cmp "$WORKDIR/source/rand3" "$WORKDIR/mount/rand3"
++    cmp "$WORKDIR/source/z1 with spaces" "$WORKDIR/mount/z1 with spaces"
++
++    echo "Parallel md5sum..."
++    md5sum "$WORKDIR"/mount/* >"$WORKDIR/md5sums"
++    split -l1 "$WORKDIR/md5sums" "$WORKDIR/sumpiece"
++    echo "$WORKDIR"/sumpiece* | xargs -P4 -n1 md5sum -c
++
++    echo "Lookup tests..."
++    # Look for non-existent files to exercise failed lookup path.
++    if [ -e "$WORKDIR/mount/bogus" ]; then
++        echo "Bogus existence test"
++        exit 1
++    fi
++    # Twice so we hit cache path.
++    if [ -e "$WORKDIR/mount/bogus" ]; then
++        echo "Bogus existence test #2"
++        exit 1
++    fi
++
++    SRCSZ=$(wc -c < "$WORKDIR/source/rand1")
++    MNTSZ=$(wc -c < "$WORKDIR/mount/rand1")
++    if [ "$SRCSZ" != "$MNTSZ" ]; then
++        echo "Bogus size $MNTSZ != $SRCSZ"
++        exit 1
++    fi
++
++    echo "Unmounting..."
++    sq_umount "$WORKDIR/mount"
++
++    # Only test timeouts once, it takes a long time
++    if [ -z "$did_timeout" ]; then
++        echo "Remounting with idle unmount option..."
++        $SFLL $SFLL_EXTRA_ARGS -otimeout=$IDLE_TIMEOUT "$WORKDIR/squashfs.image" "$WORKDIR/mount"
++        if ! sq_is_mountpoint "$WORKDIR/mount"; then
++            echo "Not mounted?"
++            exit 1
++        fi
++        echo "Waiting up to $(( IDLE_TIMEOUT + 10 )) seconds for idle unmount..."
++        sleep $(( IDLE_TIMEOUT + 10 ))
++        if sq_is_mountpoint "$WORKDIR/mount"; then
++            echo "FS did not idle unmount in timely way."
++            exit 1
++        fi
++
++        did_timeout=yes
++    fi
++
++    rm -f "$WORKDIR/squashfs.image"
++done
++
++echo "Success."
++exit 0
+diff --git a/tests/ll-smoke.sh.in b/tests/ll-smoke.sh.in
+index 84256267..d7ddd8a1 100755
+--- a/tests/ll-smoke.sh.in
++++ b/tests/ll-smoke.sh.in
+@@ -52,13 +52,13 @@ head -c 100000000 /dev/urandom >"$WORKDIR/source/rand3"
+ head -c 87 /dev/zero >"$WORKDIR/source/z1 with spaces"
+ 
+ for comp in $compressors; do
+-    echo "Building $comp squashfs image,.,"
++    echo "Building $comp squashfs image..."
+     mksquashfs "$WORKDIR/source" "$WORKDIR/squashfs.image" -comp $comp -no-progress
+ 
+     mkdir -p "$WORKDIR/mount"
+ 
+     echo "Mounting squashfs image..."
+-    $SFLL -f  "$WORKDIR/squashfs.image" "$WORKDIR/mount" >"$WORKDIR/squashfs_ll.log" 2>&1 &
++    $SFLL -f $SFLL_EXTRA_ARGS "$WORKDIR/squashfs.image" "$WORKDIR/mount" >"$WORKDIR/squashfs_ll.log" 2>&1 &
+     # Wait up to 5 seconds to be mounted. TSAN builds can take some time to mount.
+     for _ in $(seq 5); do
+     if sq_is_mountpoint "$WORKDIR/mount"; then
+@@ -119,7 +119,7 @@ for comp in $compressors; do
+     # Only test timeouts once, it takes a long time
+     if [ -z "$did_timeout" ]; then
+         echo "Remounting with idle unmount option..."
+-        $SFLL -otimeout=$IDLE_TIMEOUT "$WORKDIR/squashfs.image" "$WORKDIR/mount"
++        $SFLL $SFLL_EXTRA_ARGS -otimeout=$IDLE_TIMEOUT "$WORKDIR/squashfs.image" "$WORKDIR/mount"
+         if ! sq_is_mountpoint "$WORKDIR/mount"; then
+             echo "Not mounted?"
+             exit 1
+
+From 069e8f802481af0636fabef46bec75de1992b220 Mon Sep 17 00:00:00 2001
+From: Kevin Vigor <kvigor@gmail.com>
+Date: Mon, 23 May 2022 14:55:25 -0600
+Subject: [PATCH 3/3] Enable lazy umount on SIGTERM.
+
+libfuse sets SIGTERM signal handler to exit immediately. This is very
+unfortunate if any other processes are still using the filesystem.
+
+Teach squashfuse_ll to respond to SIGTERM with lazy umount. We cannot
+directly call umount2() API from the signal handler, since it is not
+signal safe, but we can fork/exec fusermount3 (yay posix?). This is
+also a win because fusermount is suid, enabling non-privileged users
+to umount. Note that normal libfuse umount uses same strategy when
+running as non-root.
+
+Note that this must be explicitly enabled at configure time with
+--enable-sigterm-handler, and it is only tested on linux.
+---
+ Makefile.am             |  3 ++
+ configure.ac            |  9 +++++
+ ll_main.c               | 87 +++++++++++++++++++++++++++++++++++++++++
+ tests/umount-test.sh.in | 85 ++++++++++++++++++++++++++++++++++++++++
+ 4 files changed, 184 insertions(+)
+ create mode 100755 tests/umount-test.sh.in
+
+diff --git a/Makefile.am b/Makefile.am
+index 17b01be4..67c17cde 100644
+--- a/Makefile.am
++++ b/Makefile.am
+@@ -111,6 +111,9 @@ if MULTITHREADED
+ # that case.
+ TESTS += tests/ll-smoke-singlethreaded.sh
+ endif
++if SIGTERM_HANDLER
++TESTS += tests/umount-test.sh
++endif
+ check_PROGRAMS = cachetest endiantest
+ cachetest_SOURCES=tests/cachetest.c
+ cachetest_LDADD=libsquashfuse.la $(COMPRESSION_LIBS)
+diff --git a/configure.ac b/configure.ac
+index 3869075a..7cd4db77 100644
+--- a/configure.ac
++++ b/configure.ac
+@@ -91,6 +91,7 @@ AS_IF([test "x$sq_tests" = x], [sq_tests=" none"])
+ 
+ AC_SUBST([sq_mksquashfs_compressors])
+ AC_CONFIG_FILES([tests/ll-smoke.sh],[chmod +x tests/ll-smoke.sh])
++AC_CONFIG_FILES([tests/umount-test.sh],[chmod +x tests/umount-test.sh])
+ 
+ 
+ AS_IF([test "x$sq_high_level$sq_low_level$sq_demo" = xnonono],
+@@ -104,6 +105,14 @@ AC_ARG_ENABLE([multithreading],
+     ])
+ AM_CONDITIONAL([MULTITHREADED], [test x$enable_multithreading = xyes])
+ 
++AC_ARG_ENABLE([sigterm-handler],
++	AS_HELP_STRING([--enable-sigterm-handler], [enable lazy umount on SIGTERM in low-level FUSE driver]),
++	[
++    AC_CHECK_HEADER([linux/version.h], , [], AC_MSG_ERROR([linux host required for sigterm-handler.]))
++    AC_DEFINE(SQFS_SIGTERM_HANDLER, 1, [Enable lazy umount on SIGTERM in low-level FUSE driver])
++    ])
++AM_CONDITIONAL([SIGTERM_HANDLER], [test x$enable_sigterm_handler = xyes])
++
+ AC_SUBST([sq_decompressors])
+ AC_SUBST([sq_high_level])
+ AC_SUBST([sq_low_level])
+diff --git a/ll_main.c b/ll_main.c
+index 22302085..0f956b5d 100644
+--- a/ll_main.c
++++ b/ll_main.c
+@@ -37,6 +37,90 @@
+ #include <signal.h>
+ #include <unistd.h>
+ 
++
++#if defined(SQFS_SIGTERM_HANDLER)
++#include <sys/utsname.h>
++#include <linux/version.h>
++static bool kernel_version_at_least(unsigned required_major,
++                                    unsigned required_minor,
++                                    unsigned required_micro) {
++    struct utsname info;
++
++    if (uname(&info) >= 0) {
++        unsigned major, minor, micro;
++
++        if (sscanf(info.release, "%u.%u.%u", &major, &minor, &micro) == 3) {
++            return KERNEL_VERSION(major, minor, micro) >=
++                KERNEL_VERSION(required_major, required_minor, required_micro);
++        }
++    }
++    return false;
++}
++
++/* libfuse's default SIGTERM handler (set up in fuse_set_signal_handlers())
++ * immediately calls fuse_session_exit(), which shuts down the filesystem
++ * even if there are active users. This leads to nastiness if other processes
++ * still depend on the filesystem.
++ *
++ * So: we respond to SIGTERM by starting a lazy unmount. This is done
++ * by exec'ing fusermount3, which works properly for unpriviledged
++ * users (we cannot use umount2() syscall because it is not signal safe;
++ * fork() and exec(), amazingly, are).
++ *
++ * If we fail to start the lazy umount, we signal ourself with SIGINT,
++ * which falls back to the old behavior of exiting ASAP.
++ */
++static const char *g_mount_point = NULL;
++static void sigterm_handler(int signum) {
++	/* Unfortunately, lazy umount of in-use fuse filesystem triggers
++	 * kernel bug on kernels < 5.2, Fixed by kernel commit
++	 * e8f3bd773d22f488724dffb886a1618da85c2966 in 5.2.
++	 */
++	if (g_mount_point && kernel_version_at_least(5,2,0)) {
++        int pid = fork();
++        if (pid == 0) {
++            /* child process: disassociate ourself from parent so
++             * we do not become zombie (as parent does not waitpid()).
++             */
++            pid_t parent = getppid();
++            setsid();
++            execl("/bin/fusermount3", "fusermount3",
++                  "-u", "-q", "-z", "--", g_mount_point, NULL);
++            execlp("fusermount3", "fusermount3",
++                   "-u", "-q", "-z", "--", g_mount_point, NULL);
++            /* if we get here, we can't run fusermount,
++             * kill the original process with a harshness.
++             */
++            kill(parent, SIGINT);
++            _exit(0);
++        } else if (pid > 0) {
++            /* parent process: nothing to do, murderous child will do us
++             * in one way or another.
++             */
++            return;
++        }
++    }
++    /* If we get here, we have failed to lazy unmount for whatever reason,
++     * kill ourself more brutally.
++     */
++    kill(getpid(), SIGINT);
++}
++
++static void set_sigterm_handler(const char *mountpoint) {
++    struct sigaction sa;
++
++    g_mount_point = mountpoint;
++    memset(&sa, 0, sizeof(sa));
++    sa.sa_handler = sigterm_handler;
++    sigemptyset(&(sa.sa_mask));
++    sa.sa_flags = SA_RESTART;
++
++    if (sigaction(SIGTERM, &sa, NULL) == -1) {
++        perror("sigaction(SIGTERM)");
++    }
++}
++#endif /* SQFS_SIGTERM_HANDLER */
++
+ int main(int argc, char *argv[]) {
+ 	struct fuse_args args;
+ 	sqfs_opts opts;
+@@ -139,6 +223,9 @@ int main(int argc, char *argv[]) {
+                         ll) == SQFS_OK) {
+ 			if (sqfs_ll_daemonize(fuse_cmdline_opts.foreground) != -1) {
+ 				if (fuse_set_signal_handlers(ch.session) != -1) {
++#if defined(SQFS_SIGTERM_HANDLER)
++					set_sigterm_handler(fuse_cmdline_opts.mountpoint);
++#endif
+ 					if (opts.idle_timeout_secs) {
+ 						setup_idle_timeout(ch.session, opts.idle_timeout_secs);
+ 					}
+diff --git a/tests/umount-test.sh.in b/tests/umount-test.sh.in
+new file mode 100755
+index 00000000..06fed533
+--- /dev/null
++++ b/tests/umount-test.sh.in
+@@ -0,0 +1,85 @@
++#!/bin/bash
++
++. "tests/lib.sh"
++
++SFLL=${1:-./squashfuse_ll}         # The squashfuse_ll binary.
++TIMEOUT=20
++
++case @build_os@ in
++    linux*)
++        ;;
++    *)
++        echo "This test is only enabled on linux hosts."
++        exit 0
++        ;;
++esac
++
++function cleanup {
++    set +e
++    if [[ -n "$TAIL_PID" ]]; then
++        kill "$TAIL_PID"
++    fi
++    @sq_fusermount@ -u "$MNTDIR" >& /dev/null
++    rm -rf "$WORKDIR"
++}
++
++set -e
++WORKDIR=$(mktemp -d)
++MNTDIR="$WORKDIR/mountpoint"
++mkdir -p "$MNTDIR"
++mkdir -p "$WORKDIR/source"
++trap cleanup EXIT
++
++# Make a tiny squashfs filesystem.
++echo "Hello world" >"$WORKDIR/source/hello"
++mksquashfs "$WORKDIR/source" "$WORKDIR/squashfs.image" -comp zstd -no-progress >& /dev/null
++
++# Mount it.
++$SFLL "$WORKDIR/squashfs.image" "$MNTDIR"
++SFPID=$(pgrep -f "squashfuse_ll.*$MNTDIR")
++
++if ! [[ -d /proc/$SFPID ]]; then
++    echo "squashfuse process missing"
++    exit 1
++fi
++if ! grep -q "$MNTDIR" /proc/mounts; then
++    echo "mount missing."
++    exit 1
++fi
++
++# background a task to hold a file open from the image.
++tail -f "${MNTDIR}/hello" >/dev/null &
++TAIL_PID=$!
++
++# SIGTERM the squashfuse process.
++kill -15 "$SFPID"
++
++# Now we expect the mountpoint to disappear due to lazy umount.
++if ! timeout $TIMEOUT bash -c \
++    "while grep -q $MNTDIR /proc/mounts; do \
++        sleep 1;
++     done"; then
++     echo "$MNTDIR did not dismount in response to SIGTERM."
++     exit 1
++fi
++
++# but the process should remain alive, because of the background task.
++if ! [[ -d /proc/$SFPID ]]; then
++    echo "squashfuse process missing"
++    exit 1
++fi
++
++# Now kill the background process.
++kill $TAIL_PID
++TAIL_PID=
++
++# Now we expect the process to die.
++if ! timeout $TIMEOUT bash -c \
++    "while [[ -d /proc/$SFPID ]]; do \
++        sleep 1;
++     done"; then
++     echo "squashfuse process did not die once filesystem was released."
++     exit 1
++fi
++
++echo "Success."

diff --git a/apptainer.spec b/apptainer.spec
index 3654a5f..ecfdaba 100644
--- a/apptainer.spec
+++ b/apptainer.spec
@@ -31,11 +31,14 @@
 
 # This can be slightly different than %%{version}.
 # For example, it has dash instead of tilde for release candidates.
-%global package_version 1.1.0-rc.2
+%global package_version 1.1.0-rc.3
+
+# Uncomment this to include a multithreaded version of squashfuse_ll
+%global squashfuse_version 0.1.105
 
 Summary: Application and environment virtualization
 Name: apptainer
-Version: 1.1.0~rc.2
+Version: 1.1.0~rc.3
 Release: 1%{?dist}
 # See LICENSE.md for first party code (BSD-3-Clause and LBNL BSD)
 # See LICENSE_THIRD_PARTY.md for incorporated code (ASL 2.0)
@@ -44,11 +47,11 @@ Release: 1%{?dist}
 License: BSD and LBNL BSD and ASL 2.0
 URL: https://apptainer.org
 Source: https://github.com/%{name}/%{name}/releases/download/v%{package_version}/%{name}-%{package_version}.tar.gz
-%if 0%{?fedora} == 35
-# Include golang source because f35 does not have a new enough version
-Source1: go1.17.12.src.tar.gz
-%endif
 
+%if "%{?squashfuse_version}" != ""
+Source10: https://github.com/vasi/squashfuse/archive/%{squashfuse_version}/squashfuse-%{squashfuse_version}.tar.gz
+Patch10: https://github.com/vasi/squashfuse/pull/70.patch
+%endif
 # The singularity package was renamed to apptainer after version 3.8.x.
 # The apptainer package reset numbering at 1.0.0, and some singularity
 #  packages are in epoch 1, so Provide corresponding singularity versions
@@ -75,13 +78,22 @@ Requires: squashfs
 Requires: squashfs-tools
 %endif
 BuildRequires: cryptsetup
+%if "%{?squashfuse_version}" != ""
+BuildRequires: autoconf
+BuildRequires: automake
+BuildRequires: libtool
+BuildRequires: pkgconfig
+BuildRequires: fuse3-devel
+BuildRequires: zlib-devel
+%endif
 Requires: squashfuse
 Requires: fakeroot
 Requires: fuse-overlayfs
+Requires: e2fsprogs
+# Uncomment this for the epel build, but we don't want it for the Apptainer
+#  release build because there the same rpm is shared across OS versions
 %if 0%{?el7}
 Requires: fuse2fs
-%else
-Requires: e2fsprogs
 %endif
 
 %description
@@ -96,9 +108,25 @@ Requires: %{name} = %{version}-%{release}
 Provides the optional setuid-root portion of Apptainer.
 
 %prep
+%if "%{?squashfuse_version}" != ""
+# the default directory for other steps is where the %prep section ends
+# so do main package last
+%setup -b 10 -n squashfuse-%{squashfuse_version}
+%patch -P 10 -p1
+%setup -n %{name}-%{package_version}
+%else
 %autosetup -n %{name}-%{package_version}
+%endif
 
 %build
+%if "%{?squashfuse_version}" != ""
+pushd ../squashfuse-%{squashfuse_version}
+./autogen.sh
+FLAGS=-std=c99 ./configure --enable-multithreading
+%make_build squashfuse_ll
+popd
+%endif
+
 %if "%{?SOURCE1}" != ""
 GOVERSION="$(echo %SOURCE1|sed 's,.*/,,;s/go//;s/\.src.*//')"
 if ! ./mlocal/scripts/check-min-go-version go $GOVERSION; then
@@ -138,6 +166,20 @@ export PATH=$PWD/go/bin:$PATH
 %endif
 
 %make_install -C builddir V=
+%if "%{?squashfuse_version}" != ""
+install -m 755 ../squashfuse-%{squashfuse_version}/squashfuse_ll %{buildroot}%{_libexecdir}/%{name}/bin/squashfuse_ll
+%endif
+
+%if 0%{?el7}
+# Check for fuse2fs only as a pre-install so that an rpm built on el7 can
+# be used on el8 & el9.  Only el7 has a fuse2fs package, the others have 
+# the fuse2fs program in the e2fsprogs package.
+%pre
+if [ ! -f /usr/bin/fuse2fs ] && [ ! -f /usr/sbin/fuse2fs ]; then
+	echo "fuse2fs not found, please yum install /usr/*bin/fuse2fs from epel" >&2
+	exit 1
+fi
+%endif
 
 %posttrans
 # clean out empty directories under /etc/singularity
@@ -154,6 +196,9 @@ rmdir %{_sysconfdir}/singularity/* %{_sysconfdir}/singularity 2>/dev/null || tru
 %dir %{_libexecdir}/%{name}
 %dir %{_libexecdir}/%{name}/bin
 %{_libexecdir}/%{name}/bin/starter
+%if "%{?squashfuse_version}" != ""
+%{_libexecdir}/%{name}/bin/squashfuse_ll
+%endif
 %{_libexecdir}/%{name}/cni
 %{_libexecdir}/%{name}/lib
 %dir %{_sysconfdir}/%{name}
@@ -174,6 +219,13 @@ rmdir %{_sysconfdir}/singularity/* %{_sysconfdir}/singularity 2>/dev/null || tru
 %attr(4755, root, root) %{_libexecdir}/%{name}/bin/starter-suid
 
 %changelog
+* Tue Sep 06 2022 Dave Dykstra <dwd@fedoraproject.org> - 1.1.0-rc.3
+- Update to upstream 1.1.0~rc.3.  Uncomment setting squashfuse_version and
+  the requiring of fuse2fs on el7.
+
+* Thu Sep 01 2022 Dave Dykstra <dwd@fedoraproject.org> - 1.1.0-rc.2+37-g1f91ff3
+- Test build
+
 * Wed Aug 17 2022 Dave Dykstra <dwd@fedoraproject.org> - 1.1.0~rc.2
 - Update to upstream 1.1.0~rc.2.  Remove customizations put into
   1.1.0-rc.1 packaging except for f35 inclusion of golang source.

diff --git a/sources b/sources
index d521293..f38197c 100644
--- a/sources
+++ b/sources
@@ -1 +1,2 @@
-SHA512 (apptainer-1.1.0-rc.2.tar.gz) = f4335439fa0fa7fc061deab359ee6edcc5e49f6a392216ee40d3c5bf07a00dabdfc3cee48a845c04017ab0ca7a3311e26c369669f488a1661cee2da2fd7b0741
+SHA512 (apptainer-1.1.0-rc.3.tar.gz) = 80848f95703ccbd8d2a12b7784ffe7521b9606490e5b183e64cf8dc7dfed471246aac6766e7c3f68adc871198fa26c74f7fa2d81f2dab3e0031440c71540bf0f
+SHA512 (squashfuse-0.1.105.tar.gz) = 6bf18575fd4732f3b0e4530902f2556859c2efbbd781a31bdbf97fe6601412dc750a80354a8ab08a3132d4f8680ea5ff0dd04440f29243906f1017aa6c70bede