Example File Systems

pyfuse3 comes with several example file systems in the examples directory of the release tarball. For completeness, these examples are also included here.

Single-file, Read-only File System

(shipped as examples/lltest.py)

  1#!/usr/bin/env python3
  2# -*- coding: utf-8 -*-
  3'''
  4hello.py - Example file system for pyfuse3.
  5
  6This program presents a static file system containing a single file.
  7
  8Copyright © 2015 Nikolaus Rath <Nikolaus.org>
  9Copyright © 2015 Gerion Entrup.
 10
 11Permission is hereby granted, free of charge, to any person obtaining a copy of
 12this software and associated documentation files (the "Software"), to deal in
 13the Software without restriction, including without limitation the rights to
 14use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
 15the Software, and to permit persons to whom the Software is furnished to do so.
 16
 17THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 18IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 19FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 20COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 21IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 22CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 23'''
 24
 25import errno
 26import logging
 27import os
 28import stat
 29from argparse import ArgumentParser, Namespace
 30from typing import cast
 31
 32import trio
 33
 34import pyfuse3
 35from pyfuse3 import EntryAttributes, FileHandleT, FileInfo, InodeT, ReaddirToken, RequestContext
 36
 37try:
 38    import faulthandler
 39except ImportError:
 40    pass
 41else:
 42    faulthandler.enable()
 43
 44log = logging.getLogger(__name__)
 45
 46
 47class TestFs(pyfuse3.Operations):
 48    def __init__(self) -> None:
 49        super(TestFs, self).__init__()
 50        self.hello_name = b"message"
 51        self.hello_inode = cast(InodeT, pyfuse3.ROOT_INODE + 1)
 52        self.hello_data = b"hello world\n"
 53
 54    async def getattr(self, inode: InodeT, ctx: RequestContext | None = None) -> EntryAttributes:
 55        entry = EntryAttributes()
 56        if inode == pyfuse3.ROOT_INODE:
 57            entry.st_mode = stat.S_IFDIR | 0o755
 58            entry.st_size = 0
 59        elif inode == self.hello_inode:
 60            entry.st_mode = stat.S_IFREG | 0o644
 61            entry.st_size = len(self.hello_data)
 62        else:
 63            raise pyfuse3.FUSEError(errno.ENOENT)
 64
 65        stamp = int(1438467123.985654 * 1e9)
 66        entry.st_atime_ns = stamp
 67        entry.st_ctime_ns = stamp
 68        entry.st_mtime_ns = stamp
 69        entry.st_gid = os.getgid()
 70        entry.st_uid = os.getuid()
 71        entry.st_ino = inode
 72
 73        return entry
 74
 75    async def lookup(
 76        self, parent_inode: InodeT, name: bytes, ctx: RequestContext
 77    ) -> EntryAttributes:
 78        if parent_inode != pyfuse3.ROOT_INODE or name != self.hello_name:
 79            raise pyfuse3.FUSEError(errno.ENOENT)
 80        return await self.getattr(self.hello_inode, ctx)
 81
 82    async def opendir(self, inode: InodeT, ctx: RequestContext) -> FileHandleT:
 83        if inode != pyfuse3.ROOT_INODE:
 84            raise pyfuse3.FUSEError(errno.ENOENT)
 85        # For simplicity, we use the inode as file handle
 86        return FileHandleT(inode)
 87
 88    async def readdir(self, fh: FileHandleT, start_id: int, token: ReaddirToken) -> None:
 89        assert fh == pyfuse3.ROOT_INODE
 90
 91        # only one entry
 92        if start_id == 0:
 93            pyfuse3.readdir_reply(token, self.hello_name, await self.getattr(self.hello_inode), 1)
 94        return
 95
 96    async def open(self, inode: InodeT, flags: int, ctx: RequestContext) -> FileInfo:
 97        if inode != self.hello_inode:
 98            raise pyfuse3.FUSEError(errno.ENOENT)
 99        if flags & os.O_RDWR or flags & os.O_WRONLY:
100            raise pyfuse3.FUSEError(errno.EACCES)
101        # For simplicity, we use the inode as file handle
102        return FileInfo(fh=FileHandleT(inode))
103
104    async def read(self, fh: FileHandleT, off: int, size: int) -> bytes:
105        assert fh == self.hello_inode
106        return self.hello_data[off : off + size]
107
108
109def init_logging(debug: bool = False) -> None:
110    formatter = logging.Formatter(
111        '%(asctime)s.%(msecs)03d %(threadName)s: [%(name)s] %(message)s',
112        datefmt="%Y-%m-%d %H:%M:%S",
113    )
114    handler = logging.StreamHandler()
115    handler.setFormatter(formatter)
116    root_logger = logging.getLogger()
117    if debug:
118        handler.setLevel(logging.DEBUG)
119        root_logger.setLevel(logging.DEBUG)
120    else:
121        handler.setLevel(logging.INFO)
122        root_logger.setLevel(logging.INFO)
123    root_logger.addHandler(handler)
124
125
126def parse_args() -> Namespace:
127    '''Parse command line'''
128
129    parser = ArgumentParser()
130
131    parser.add_argument('mountpoint', type=str, help='Where to mount the file system')
132    parser.add_argument(
133        '--debug', action='store_true', default=False, help='Enable debugging output'
134    )
135    parser.add_argument(
136        '--debug-fuse', action='store_true', default=False, help='Enable FUSE debugging output'
137    )
138    return parser.parse_args()
139
140
141def main() -> None:
142    options = parse_args()
143    init_logging(options.debug)
144
145    testfs = TestFs()
146    fuse_options = set(pyfuse3.default_options)
147    fuse_options.add('fsname=hello')
148    if options.debug_fuse:
149        fuse_options.add('debug')
150    pyfuse3.init(testfs, options.mountpoint, fuse_options)
151    try:
152        trio.run(pyfuse3.main)
153    except:
154        pyfuse3.close(unmount=False)
155        raise
156
157    pyfuse3.close()
158
159
160if __name__ == '__main__':
161    main()

In-memory File System

(shipped as examples/tmpfs.py)

  1#!/usr/bin/env python3
  2# -*- coding: utf-8 -*-
  3'''
  4tmpfs.py - Example file system for pyfuse3.
  5
  6This file system stores all data in memory.
  7
  8Copyright © 2013 Nikolaus Rath <Nikolaus.org>
  9
 10Permission is hereby granted, free of charge, to any person obtaining a copy of
 11this software and associated documentation files (the "Software"), to deal in
 12the Software without restriction, including without limitation the rights to
 13use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
 14the Software, and to permit persons to whom the Software is furnished to do so.
 15
 16THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 17IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 18FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 19COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 20IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 21CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 22'''
 23
 24import errno
 25import logging
 26import os
 27import sqlite3
 28import stat
 29from argparse import ArgumentParser, Namespace
 30from collections import defaultdict
 31from time import time
 32from typing import Any, cast
 33
 34import trio
 35
 36import pyfuse3
 37from pyfuse3 import (
 38    EntryAttributes,
 39    FileHandleT,
 40    FileInfo,
 41    FUSEError,
 42    InodeT,
 43    ReaddirToken,
 44    RequestContext,
 45    SetattrFields,
 46    StatvfsData,
 47)
 48
 49try:
 50    import faulthandler
 51except ImportError:
 52    pass
 53else:
 54    faulthandler.enable()
 55
 56log = logging.getLogger()
 57
 58
 59class Operations(pyfuse3.Operations):
 60    '''An example filesystem that stores all data in memory
 61
 62    This is a very simple implementation with terrible performance.
 63    Don't try to store significant amounts of data. Also, there are
 64    some other flaws that have not been fixed to keep the code easier
 65    to understand:
 66
 67    * atime, mtime and ctime are not updated
 68    * generation numbers are not supported
 69    * lookup counts are not maintained
 70    '''
 71
 72    enable_writeback_cache = True
 73
 74    def __init__(self) -> None:
 75        super(Operations, self).__init__()
 76        self.db: sqlite3.Connection = sqlite3.connect(':memory:')
 77        self.db.text_factory = str
 78        self.db.row_factory = sqlite3.Row
 79        self.cursor: sqlite3.Cursor = self.db.cursor()
 80        self.inode_open_count: defaultdict[InodeT, int] = defaultdict(int)
 81        self.init_tables()
 82
 83    def init_tables(self) -> None:
 84        '''Initialize file system tables'''
 85
 86        self.cursor.execute("""
 87        CREATE TABLE inodes (
 88            id        INTEGER PRIMARY KEY,
 89            uid       INT NOT NULL,
 90            gid       INT NOT NULL,
 91            mode      INT NOT NULL,
 92            mtime_ns  INT NOT NULL,
 93            atime_ns  INT NOT NULL,
 94            ctime_ns  INT NOT NULL,
 95            target    BLOB(256) ,
 96            size      INT NOT NULL DEFAULT 0,
 97            rdev      INT NOT NULL DEFAULT 0,
 98            data      BLOB
 99        )
100        """)
101
102        self.cursor.execute("""
103        CREATE TABLE contents (
104            rowid     INTEGER PRIMARY KEY AUTOINCREMENT,
105            name      BLOB(256) NOT NULL,
106            inode     INT NOT NULL REFERENCES inodes(id),
107            parent_inode INT NOT NULL REFERENCES inodes(id),
108
109            UNIQUE (name, parent_inode)
110        )""")
111
112        # Insert root directory
113        now_ns = int(time() * 1e9)
114        self.cursor.execute(
115            "INSERT INTO inodes (id,mode,uid,gid,mtime_ns,atime_ns,ctime_ns) "
116            "VALUES (?,?,?,?,?,?,?)",
117            (
118                pyfuse3.ROOT_INODE,
119                stat.S_IFDIR
120                | stat.S_IRUSR
121                | stat.S_IWUSR
122                | stat.S_IXUSR
123                | stat.S_IRGRP
124                | stat.S_IXGRP
125                | stat.S_IROTH
126                | stat.S_IXOTH,
127                os.getuid(),
128                os.getgid(),
129                now_ns,
130                now_ns,
131                now_ns,
132            ),
133        )
134        self.cursor.execute(
135            "INSERT INTO contents (name, parent_inode, inode) VALUES (?,?,?)",
136            (b'..', pyfuse3.ROOT_INODE, pyfuse3.ROOT_INODE),
137        )
138
139    def get_row(self, *a: Any, **kw: Any) -> sqlite3.Row:
140        self.cursor.execute(*a, **kw)
141        try:
142            row = next(self.cursor)
143        except StopIteration:
144            raise NoSuchRowError()
145        try:
146            next(self.cursor)
147        except StopIteration:
148            pass
149        else:
150            raise NoUniqueValueError()
151
152        return row
153
154    async def lookup(
155        self, parent_inode: InodeT, name: bytes, ctx: RequestContext
156    ) -> EntryAttributes:
157        if name == b'.':
158            inode = parent_inode
159        elif name == b'..':
160            inode = self.get_row("SELECT * FROM contents WHERE inode=?", (parent_inode,))[
161                'parent_inode'
162            ]
163        else:
164            try:
165                inode = self.get_row(
166                    "SELECT * FROM contents WHERE name=? AND parent_inode=?", (name, parent_inode)
167                )['inode']
168            except NoSuchRowError:
169                raise (pyfuse3.FUSEError(errno.ENOENT))
170
171        return await self.getattr(InodeT(inode), ctx)
172
173    async def getattr(self, inode: InodeT, ctx: RequestContext | None = None) -> EntryAttributes:
174        try:
175            row = self.get_row("SELECT * FROM inodes WHERE id=?", (inode,))
176        except NoSuchRowError:
177            raise (pyfuse3.FUSEError(errno.ENOENT))
178
179        entry = EntryAttributes()
180        entry.st_ino = inode
181        entry.generation = 0
182        entry.entry_timeout = 300
183        entry.attr_timeout = 300
184        entry.st_mode = row['mode']
185        entry.st_nlink = self.get_row("SELECT COUNT(inode) FROM contents WHERE inode=?", (inode,))[
186            0
187        ]
188        entry.st_uid = row['uid']
189        entry.st_gid = row['gid']
190        entry.st_rdev = row['rdev']
191        entry.st_size = row['size']
192
193        entry.st_blksize = 512
194        entry.st_blocks = 1
195        entry.st_atime_ns = row['atime_ns']
196        entry.st_mtime_ns = row['mtime_ns']
197        entry.st_ctime_ns = row['ctime_ns']
198
199        return entry
200
201    async def readlink(self, inode: InodeT, ctx: RequestContext) -> bytes:
202        return self.get_row('SELECT * FROM inodes WHERE id=?', (inode,))['target']
203
204    async def opendir(self, inode: InodeT, ctx: RequestContext) -> FileHandleT:
205        # For simplicity, we use the inode as file handle
206        return FileHandleT(inode)
207
208    async def readdir(self, fh: FileHandleT, start_id: int, token: ReaddirToken) -> None:
209        if start_id == 0:
210            off = -1
211        else:
212            off = start_id
213
214        cursor2 = self.db.cursor()
215        cursor2.execute(
216            "SELECT * FROM contents WHERE parent_inode=? AND rowid > ? ORDER BY rowid", (fh, off)
217        )
218
219        for row in cursor2:
220            pyfuse3.readdir_reply(
221                token, row['name'], await self.getattr(InodeT(row['inode'])), row['rowid']
222            )
223
224    async def unlink(self, parent_inode: InodeT, name: bytes, ctx: RequestContext) -> None:
225        entry = await self.lookup(parent_inode, name, ctx)
226
227        if stat.S_ISDIR(entry.st_mode):
228            raise pyfuse3.FUSEError(errno.EISDIR)
229
230        self._remove(parent_inode, name, entry)
231
232    async def rmdir(self, parent_inode: InodeT, name: bytes, ctx: RequestContext) -> None:
233        entry = await self.lookup(parent_inode, name, ctx)
234
235        if not stat.S_ISDIR(entry.st_mode):
236            raise pyfuse3.FUSEError(errno.ENOTDIR)
237
238        self._remove(parent_inode, name, entry)
239
240    def _remove(self, parent_inode: InodeT, name: bytes, entry: EntryAttributes) -> None:
241        if (
242            self.get_row("SELECT COUNT(inode) FROM contents WHERE parent_inode=?", (entry.st_ino,))[
243                0
244            ]
245            > 0
246        ):
247            raise pyfuse3.FUSEError(errno.ENOTEMPTY)
248
249        self.cursor.execute(
250            "DELETE FROM contents WHERE name=? AND parent_inode=?", (name, parent_inode)
251        )
252
253        if entry.st_nlink == 1 and entry.st_ino not in self.inode_open_count:
254            self.cursor.execute("DELETE FROM inodes WHERE id=?", (entry.st_ino,))
255
256    async def symlink(
257        self, parent_inode: InodeT, name: bytes, target: bytes, ctx: RequestContext
258    ) -> EntryAttributes:
259        mode = (
260            stat.S_IFLNK
261            | stat.S_IRUSR
262            | stat.S_IWUSR
263            | stat.S_IXUSR
264            | stat.S_IRGRP
265            | stat.S_IWGRP
266            | stat.S_IXGRP
267            | stat.S_IROTH
268            | stat.S_IWOTH
269            | stat.S_IXOTH
270        )
271        return await self._create(parent_inode, name, mode, ctx, target=target)
272
273    async def rename(
274        self,
275        parent_inode_old: InodeT,
276        name_old: bytes,
277        parent_inode_new: InodeT,
278        name_new: bytes,
279        flags: int,
280        ctx: RequestContext,
281    ) -> None:
282        if flags != 0:
283            raise FUSEError(errno.EINVAL)
284
285        entry_old = await self.lookup(parent_inode_old, name_old, ctx)
286
287        entry_new = None
288        try:
289            entry_new = await self.lookup(
290                parent_inode_new,
291                name_new if isinstance(name_new, bytes) else name_new.encode(),
292                ctx,
293            )
294        except pyfuse3.FUSEError as exc:
295            if exc.errno != errno.ENOENT:
296                raise
297
298        if entry_new is not None:
299            self._replace(
300                parent_inode_old, name_old, parent_inode_new, name_new, entry_old, entry_new
301            )
302        else:
303            self.cursor.execute(
304                "UPDATE contents SET name=?, parent_inode=? WHERE name=? AND parent_inode=?",
305                (name_new, parent_inode_new, name_old, parent_inode_old),
306            )
307
308    def _replace(
309        self,
310        parent_inode_old: InodeT,
311        name_old: bytes,
312        parent_inode_new: InodeT,
313        name_new: bytes,
314        entry_old: EntryAttributes,
315        entry_new: EntryAttributes,
316    ) -> None:
317        if (
318            self.get_row(
319                "SELECT COUNT(inode) FROM contents WHERE parent_inode=?", (entry_new.st_ino,)
320            )[0]
321            > 0
322        ):
323            raise pyfuse3.FUSEError(errno.ENOTEMPTY)
324
325        self.cursor.execute(
326            "UPDATE contents SET inode=? WHERE name=? AND parent_inode=?",
327            (entry_old.st_ino, name_new, parent_inode_new),
328        )
329        self.db.execute(
330            'DELETE FROM contents WHERE name=? AND parent_inode=?', (name_old, parent_inode_old)
331        )
332
333        if entry_new.st_nlink == 1 and entry_new.st_ino not in self.inode_open_count:
334            self.cursor.execute("DELETE FROM inodes WHERE id=?", (entry_new.st_ino,))
335
336    async def link(
337        self, inode: InodeT, new_parent_inode: InodeT, new_name: bytes, ctx: RequestContext
338    ) -> EntryAttributes:
339        entry_p = await self.getattr(new_parent_inode, ctx)
340        if entry_p.st_nlink == 0:
341            log.warning(
342                'Attempted to create entry %s with unlinked parent %d', new_name, new_parent_inode
343            )
344            raise FUSEError(errno.EINVAL)
345
346        self.cursor.execute(
347            "INSERT INTO contents (name, inode, parent_inode) VALUES(?,?,?)",
348            (new_name, inode, new_parent_inode),
349        )
350
351        return await self.getattr(inode, ctx)
352
353    async def setattr(
354        self,
355        inode: InodeT,
356        attr: EntryAttributes,
357        fields: SetattrFields,
358        fh: FileHandleT | None,
359        ctx: RequestContext,
360    ) -> EntryAttributes:
361        if fields.update_size:
362            data = self.get_row('SELECT data FROM inodes WHERE id=?', (inode,))[0]
363            if data is None:
364                data = b''
365            if len(data) < attr.st_size:
366                data = data + b'\0' * (attr.st_size - len(data))
367            else:
368                data = data[: attr.st_size]
369            self.cursor.execute(
370                'UPDATE inodes SET data=?, size=? WHERE id=?',
371                (memoryview(data), attr.st_size, inode),
372            )
373        if fields.update_mode:
374            self.cursor.execute('UPDATE inodes SET mode=? WHERE id=?', (attr.st_mode, inode))
375
376        if fields.update_uid:
377            self.cursor.execute('UPDATE inodes SET uid=? WHERE id=?', (attr.st_uid, inode))
378
379        if fields.update_gid:
380            self.cursor.execute('UPDATE inodes SET gid=? WHERE id=?', (attr.st_gid, inode))
381
382        if fields.update_atime:
383            self.cursor.execute(
384                'UPDATE inodes SET atime_ns=? WHERE id=?', (attr.st_atime_ns, inode)
385            )
386
387        if fields.update_mtime:
388            self.cursor.execute(
389                'UPDATE inodes SET mtime_ns=? WHERE id=?', (attr.st_mtime_ns, inode)
390            )
391
392        if fields.update_ctime:
393            self.cursor.execute(
394                'UPDATE inodes SET ctime_ns=? WHERE id=?', (attr.st_ctime_ns, inode)
395            )
396        else:
397            self.cursor.execute(
398                'UPDATE inodes SET ctime_ns=? WHERE id=?', (int(time() * 1e9), inode)
399            )
400
401        return await self.getattr(inode, ctx)
402
403    async def mknod(
404        self, parent_inode: InodeT, name: bytes, mode: int, rdev: int, ctx: RequestContext
405    ) -> EntryAttributes:
406        return await self._create(parent_inode, name, mode, ctx, rdev=rdev)
407
408    async def mkdir(
409        self, parent_inode: InodeT, name: bytes, mode: int, ctx: RequestContext
410    ) -> EntryAttributes:
411        return await self._create(parent_inode, name, mode, ctx)
412
413    async def statfs(self, ctx: RequestContext) -> StatvfsData:
414        stat_ = StatvfsData()
415
416        stat_.f_bsize = 512
417        stat_.f_frsize = 512
418
419        size = self.get_row('SELECT SUM(size) FROM inodes')[0]
420        stat_.f_blocks = size // stat_.f_frsize
421        stat_.f_bfree = max(size // stat_.f_frsize, 1024)
422        stat_.f_bavail = stat_.f_bfree
423
424        inodes = self.get_row('SELECT COUNT(id) FROM inodes')[0]
425        stat_.f_files = inodes
426        stat_.f_ffree = max(inodes, 100)
427        stat_.f_favail = stat_.f_ffree
428
429        return stat_
430
431    async def open(self, inode: InodeT, flags: int, ctx: RequestContext) -> FileInfo:
432        self.inode_open_count[inode] += 1
433
434        # For simplicity, we use the inode as file handle
435        return FileInfo(fh=FileHandleT(inode))
436
437    async def access(self, inode: InodeT, mode: int, ctx: RequestContext) -> bool:
438        # Yeah, could be a function and has unused arguments
439        # pylint: disable=R0201,W0613
440        return True
441
442    async def create(
443        self, parent_inode: InodeT, name: bytes, mode: int, flags: int, ctx: RequestContext
444    ) -> tuple[FileInfo, EntryAttributes]:
445        # pylint: disable=W0612
446        entry = await self._create(parent_inode, name, mode, ctx)
447        self.inode_open_count[entry.st_ino] += 1
448        # For simplicity, we use the inode as file handle
449        return (FileInfo(fh=FileHandleT(entry.st_ino)), entry)
450
451    async def _create(
452        self,
453        parent_inode: InodeT,
454        name: bytes,
455        mode: int,
456        ctx: RequestContext,
457        rdev: int = 0,
458        target: bytes | None = None,
459    ) -> EntryAttributes:
460        if (await self.getattr(parent_inode, ctx)).st_nlink == 0:
461            log.warning('Attempted to create entry %s with unlinked parent %d', name, parent_inode)
462            raise FUSEError(errno.EINVAL)
463
464        now_ns = int(time() * 1e9)
465        self.cursor.execute(
466            'INSERT INTO inodes (uid, gid, mode, mtime_ns, atime_ns, '
467            'ctime_ns, target, rdev) VALUES(?, ?, ?, ?, ?, ?, ?, ?)',
468            (ctx.uid, ctx.gid, mode, now_ns, now_ns, now_ns, target, rdev),
469        )
470
471        inode = cast(InodeT, self.cursor.lastrowid)
472        self.db.execute(
473            "INSERT INTO contents(name, inode, parent_inode) VALUES(?,?,?)",
474            (name, inode, parent_inode),
475        )
476        return await self.getattr(inode, ctx)
477
478    async def read(self, fh: FileHandleT, off: int, size: int) -> bytes:
479        data = self.get_row('SELECT data FROM inodes WHERE id=?', (fh,))[0]
480        if data is None:
481            data = b''
482        return data[off : off + size]
483
484    async def write(self, fh: FileHandleT, off: int, buf: bytes) -> int:
485        data = self.get_row('SELECT data FROM inodes WHERE id=?', (fh,))[0]
486        if data is None:
487            data = b''
488        data = data[:off] + buf + data[off + len(buf) :]
489
490        self.cursor.execute(
491            'UPDATE inodes SET data=?, size=? WHERE id=?', (memoryview(data), len(data), fh)
492        )
493        return len(buf)
494
495    async def release(self, fh: FileHandleT) -> None:
496        inode = cast(InodeT, fh)
497        self.inode_open_count[inode] -= 1
498
499        if self.inode_open_count[inode] == 0:
500            del self.inode_open_count[inode]
501            if (await self.getattr(inode)).st_nlink == 0:
502                self.cursor.execute("DELETE FROM inodes WHERE id=?", (inode,))
503
504
505class NoUniqueValueError(Exception):
506    def __str__(self) -> str:
507        return 'Query generated more than 1 result row'
508
509
510class NoSuchRowError(Exception):
511    def __str__(self) -> str:
512        return 'Query produced 0 result rows'
513
514
515def init_logging(debug: bool = False) -> None:
516    formatter = logging.Formatter(
517        '%(asctime)s.%(msecs)03d %(threadName)s: [%(name)s] %(message)s',
518        datefmt="%Y-%m-%d %H:%M:%S",
519    )
520    handler = logging.StreamHandler()
521    handler.setFormatter(formatter)
522    root_logger = logging.getLogger()
523    if debug:
524        handler.setLevel(logging.DEBUG)
525        root_logger.setLevel(logging.DEBUG)
526    else:
527        handler.setLevel(logging.INFO)
528        root_logger.setLevel(logging.INFO)
529    root_logger.addHandler(handler)
530
531
532def parse_args() -> Namespace:
533    '''Parse command line'''
534
535    parser = ArgumentParser()
536
537    parser.add_argument('mountpoint', type=str, help='Where to mount the file system')
538    parser.add_argument(
539        '--debug', action='store_true', default=False, help='Enable debugging output'
540    )
541    parser.add_argument(
542        '--debug-fuse', action='store_true', default=False, help='Enable FUSE debugging output'
543    )
544
545    return parser.parse_args()
546
547
548if __name__ == '__main__':
549    options = parse_args()
550    init_logging(options.debug)
551    operations = Operations()
552
553    fuse_options = set(pyfuse3.default_options)
554    fuse_options.add('fsname=tmpfs')
555    fuse_options.discard('default_permissions')
556    if options.debug_fuse:
557        fuse_options.add('debug')
558    pyfuse3.init(operations, options.mountpoint, fuse_options)
559
560    try:
561        trio.run(pyfuse3.main)
562    except:
563        pyfuse3.close(unmount=False)
564        raise
565
566    pyfuse3.close()

Passthrough / Overlay File System

(shipped as examples/passthroughfs.py)

  1#!/usr/bin/env python3
  2'''
  3passthroughfs.py - Example file system for pyfuse3
  4
  5This file system mirrors the contents of a specified directory tree.
  6
  7Caveats:
  8
  9 * Inode generation numbers are not passed through but set to zero.
 10
 11 * Block size (st_blksize) and number of allocated blocks (st_blocks) are not
 12   passed through.
 13
 14 * Performance for large directories is not good, because the directory
 15   is always read completely.
 16
 17 * There may be a way to break-out of the directory tree.
 18
 19 * The readdir implementation is not fully POSIX compliant. If a directory
 20   contains hardlinks and is modified during a readdir call, readdir()
 21   may return some of the hardlinked files twice or omit them completely.
 22
 23 * If you delete or rename files in the underlying file system, the
 24   passthrough file system will get confused.
 25
 26Copyright ©  Nikolaus Rath <Nikolaus.org>
 27
 28Permission is hereby granted, free of charge, to any person obtaining a copy of
 29this software and associated documentation files (the "Software"), to deal in
 30the Software without restriction, including without limitation the rights to
 31use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
 32the Software, and to permit persons to whom the Software is furnished to do so.
 33
 34THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 35IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 36FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 37COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 38IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 39CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 40'''
 41
 42import errno
 43import faulthandler
 44import logging
 45import os
 46import stat as stat_m
 47import sys
 48from argparse import ArgumentParser, Namespace
 49from collections import defaultdict
 50from collections.abc import Sequence
 51from os import fsdecode, fsencode
 52from typing import cast
 53
 54import trio
 55
 56import pyfuse3
 57from pyfuse3 import (
 58    EntryAttributes,
 59    FileHandleT,
 60    FileInfo,
 61    FUSEError,
 62    InodeT,
 63    ReaddirToken,
 64    RequestContext,
 65    SetattrFields,
 66    StatvfsData,
 67)
 68
 69faulthandler.enable()
 70
 71log = logging.getLogger(__name__)
 72
 73
 74class Operations(pyfuse3.Operations):
 75    def __init__(self, source: str, enable_writeback_cache: bool = False) -> None:
 76        super().__init__()
 77        self.enable_writeback_cache = enable_writeback_cache
 78        self._inode_path_map: dict[InodeT, str | set[str]] = {pyfuse3.ROOT_INODE: source}
 79        self._lookup_cnt: defaultdict[InodeT, int] = defaultdict(lambda: 0)
 80        self._fd_inode_map: dict[int, InodeT] = dict()
 81        self._inode_fd_map: dict[InodeT, int] = dict()
 82        self._fd_open_count: dict[int, int] = dict()
 83
 84    def _inode_to_path(self, inode: InodeT) -> str:
 85        try:
 86            val = self._inode_path_map[inode]
 87        except KeyError:
 88            raise FUSEError(errno.ENOENT)
 89
 90        if isinstance(val, set):
 91            # In case of hardlinks, pick any path
 92            val = next(iter(val))
 93        return val
 94
 95    def _add_path(self, inode: InodeT, path: str) -> None:
 96        log.debug('_add_path for %d, %s', inode, path)
 97        self._lookup_cnt[inode] += 1
 98
 99        # With hardlinks, one inode may map to multiple paths.
100        if inode not in self._inode_path_map:
101            self._inode_path_map[inode] = path
102            return
103
104        val = self._inode_path_map[inode]
105        if isinstance(val, set):
106            val.add(path)
107        elif val != path:
108            self._inode_path_map[inode] = {path, val}
109
110    async def forget(self, inode_list: Sequence[tuple[InodeT, int]]) -> None:
111        for inode, nlookup in inode_list:
112            if self._lookup_cnt[inode] > nlookup:
113                self._lookup_cnt[inode] -= nlookup
114                continue
115            log.debug('forgetting about inode %d', inode)
116            assert inode not in self._inode_fd_map
117            del self._lookup_cnt[inode]
118            try:
119                del self._inode_path_map[inode]
120            except KeyError:  # may have been deleted
121                pass
122
123    async def lookup(
124        self, parent_inode: InodeT, name: bytes, ctx: RequestContext
125    ) -> EntryAttributes:
126        name_str = fsdecode(name)
127        log.debug('lookup for %s in %d', name_str, parent_inode)
128        path = os.path.join(self._inode_to_path(parent_inode), name_str)
129        attr = self._getattr(path=path)
130        if name_str != '.' and name_str != '..':
131            self._add_path(InodeT(attr.st_ino), path)
132        return attr
133
134    async def getattr(self, inode: InodeT, ctx: RequestContext | None = None) -> EntryAttributes:
135        if inode in self._inode_fd_map:
136            return self._getattr(fd=self._inode_fd_map[inode])
137        else:
138            return self._getattr(path=self._inode_to_path(inode))
139
140    def _getattr(self, path: str | None = None, fd: int | None = None) -> EntryAttributes:
141        assert fd is None or path is None
142        assert not (fd is None and path is None)
143        try:
144            if fd is None:
145                assert path is not None
146                stat = os.lstat(path)
147            else:
148                stat = os.fstat(fd)
149        except OSError as exc:
150            assert exc.errno is not None
151            raise FUSEError(exc.errno)
152
153        entry = EntryAttributes()
154        for attr in (
155            'st_ino',
156            'st_mode',
157            'st_nlink',
158            'st_uid',
159            'st_gid',
160            'st_rdev',
161            'st_size',
162            'st_atime_ns',
163            'st_mtime_ns',
164            'st_ctime_ns',
165        ):
166            setattr(entry, attr, getattr(stat, attr))
167        entry.generation = 0
168        entry.entry_timeout = 0
169        entry.attr_timeout = 0
170        entry.st_blksize = 512
171        entry.st_blocks = (entry.st_size + entry.st_blksize - 1) // entry.st_blksize
172
173        return entry
174
175    async def readlink(self, inode: InodeT, ctx: RequestContext) -> bytes:
176        path = self._inode_to_path(inode)
177        try:
178            target = os.readlink(path)
179        except OSError as exc:
180            assert exc.errno is not None
181            raise FUSEError(exc.errno)
182        return fsencode(target)
183
184    async def opendir(self, inode: InodeT, ctx: RequestContext) -> FileHandleT:
185        # For simplicity, we use the inode as file handle
186        return FileHandleT(inode)
187
188    async def readdir(self, fh: FileHandleT, start_id: int, token: ReaddirToken) -> None:
189        path = self._inode_to_path(InodeT(fh))
190        log.debug('reading %s', path)
191        entries: list[tuple[InodeT, str, EntryAttributes]] = []
192        for name in os.listdir(path):
193            if name == '.' or name == '..':
194                continue
195            attr = self._getattr(path=os.path.join(path, name))
196            entries.append((InodeT(attr.st_ino), name, attr))
197
198        log.debug('read %d entries, starting at %d', len(entries), start_id)
199
200        # This is not fully posix compatible. If there are hardlinks
201        # (two names with the same inode), we don't have a unique
202        # offset to start in between them. Note that we cannot simply
203        # count entries, because then we would skip over entries
204        # (or return them more than once) if the number of directory
205        # entries changes between two calls to readdir().
206        for ino, name, attr in sorted(entries):
207            if ino <= start_id:
208                continue
209            if not pyfuse3.readdir_reply(token, fsencode(name), attr, ino):
210                break
211            self._add_path(attr.st_ino, os.path.join(path, name))
212
213    async def unlink(self, parent_inode: InodeT, name: bytes, ctx: RequestContext) -> None:
214        name_str = fsdecode(name)
215        parent = self._inode_to_path(parent_inode)
216        path = os.path.join(parent, name_str)
217        try:
218            inode = os.lstat(path).st_ino
219            os.unlink(path)
220        except OSError as exc:
221            assert exc.errno is not None
222            raise FUSEError(exc.errno)
223        if inode in self._lookup_cnt:
224            self._forget_path(InodeT(inode), path)
225
226    async def rmdir(self, parent_inode: InodeT, name: bytes, ctx: RequestContext) -> None:
227        name_str = fsdecode(name)
228        parent = self._inode_to_path(parent_inode)
229        path = os.path.join(parent, name_str)
230        try:
231            inode = os.lstat(path).st_ino
232            os.rmdir(path)
233        except OSError as exc:
234            assert exc.errno is not None
235            raise FUSEError(exc.errno)
236        if inode in self._lookup_cnt:
237            self._forget_path(InodeT(inode), path)
238
239    def _forget_path(self, inode: InodeT, path: str) -> None:
240        log.debug('forget %s for %d', path, inode)
241        val = self._inode_path_map[inode]
242        if isinstance(val, set):
243            val.remove(path)
244            if len(val) == 1:
245                self._inode_path_map[inode] = next(iter(val))
246        else:
247            del self._inode_path_map[inode]
248
249    async def symlink(
250        self, parent_inode: InodeT, name: bytes, target: bytes, ctx: RequestContext
251    ) -> EntryAttributes:
252        name_str = fsdecode(name)
253        target_str = fsdecode(target)
254        parent = self._inode_to_path(parent_inode)
255        path = os.path.join(parent, name_str)
256        try:
257            os.symlink(target_str, path)
258            os.lchown(path, ctx.uid, ctx.gid)
259        except OSError as exc:
260            assert exc.errno is not None
261            raise FUSEError(exc.errno)
262        inode = InodeT(os.lstat(path).st_ino)
263        self._add_path(inode, path)
264        return await self.getattr(inode, ctx)
265
266    async def rename(
267        self,
268        parent_inode_old: InodeT,
269        name_old: bytes,
270        parent_inode_new: InodeT,
271        name_new: bytes,
272        flags: int,
273        ctx: RequestContext,
274    ) -> None:
275        if flags != 0:
276            raise FUSEError(errno.EINVAL)
277
278        name_old_str = fsdecode(name_old)
279        name_new_str = fsdecode(name_new)
280        parent_old = self._inode_to_path(parent_inode_old)
281        parent_new = self._inode_to_path(parent_inode_new)
282        path_old = os.path.join(parent_old, name_old_str)
283        path_new = os.path.join(parent_new, name_new_str)
284        try:
285            os.rename(path_old, path_new)
286            inode = cast(InodeT, os.lstat(path_new).st_ino)
287        except OSError as exc:
288            assert exc.errno is not None
289            raise FUSEError(exc.errno)
290        if inode not in self._lookup_cnt:
291            return
292
293        val = self._inode_path_map[inode]
294        if isinstance(val, set):
295            assert len(val) > 1
296            val.add(path_new)
297            val.remove(path_old)
298        else:
299            assert val == path_old
300            self._inode_path_map[inode] = path_new
301
302    async def link(
303        self, inode: InodeT, new_parent_inode: InodeT, new_name: bytes, ctx: RequestContext
304    ) -> EntryAttributes:
305        new_name_str = fsdecode(new_name)
306        parent = self._inode_to_path(new_parent_inode)
307        path = os.path.join(parent, new_name_str)
308        try:
309            os.link(self._inode_to_path(inode), path, follow_symlinks=False)
310        except OSError as exc:
311            assert exc.errno is not None
312            raise FUSEError(exc.errno)
313        self._add_path(inode, path)
314        return await self.getattr(inode, ctx)
315
316    async def setattr(
317        self,
318        inode: InodeT,
319        attr: EntryAttributes,
320        fields: SetattrFields,
321        fh: FileHandleT | None,
322        ctx: RequestContext,
323    ) -> EntryAttributes:
324        try:
325            if fields.update_size:
326                if fh is None:
327                    os.truncate(self._inode_to_path(inode), attr.st_size)
328                else:
329                    os.ftruncate(fh, attr.st_size)
330
331            if fields.update_mode:
332                # Under Linux, chmod always resolves symlinks so we should
333                # actually never get a setattr() request for a symbolic
334                # link.
335                assert not stat_m.S_ISLNK(attr.st_mode)
336                if fh is None:
337                    os.chmod(self._inode_to_path(inode), stat_m.S_IMODE(attr.st_mode))
338                else:
339                    os.fchmod(fh, stat_m.S_IMODE(attr.st_mode))
340
341            if fields.update_uid and fields.update_gid:
342                if fh is None:
343                    os.chown(
344                        self._inode_to_path(inode), attr.st_uid, attr.st_gid, follow_symlinks=False
345                    )
346                else:
347                    os.fchown(fh, attr.st_uid, attr.st_gid)
348
349            elif fields.update_uid:
350                if fh is None:
351                    os.chown(self._inode_to_path(inode), attr.st_uid, -1, follow_symlinks=False)
352                else:
353                    os.fchown(fh, attr.st_uid, -1)
354
355            elif fields.update_gid:
356                if fh is None:
357                    os.chown(self._inode_to_path(inode), -1, attr.st_gid, follow_symlinks=False)
358                else:
359                    os.fchown(fh, -1, attr.st_gid)
360
361            if fields.update_atime and fields.update_mtime:
362                if fh is None:
363                    os.utime(
364                        self._inode_to_path(inode),
365                        None,
366                        follow_symlinks=False,
367                        ns=(attr.st_atime_ns, attr.st_mtime_ns),
368                    )
369                else:
370                    os.utime(fh, None, ns=(attr.st_atime_ns, attr.st_mtime_ns))
371            elif fields.update_atime or fields.update_mtime:
372                # We can only set both values, so we first need to retrieve the
373                # one that we shouldn't be changing.
374                if fh is None:
375                    path = self._inode_to_path(inode)
376                    oldstat = os.stat(path, follow_symlinks=False)
377                else:
378                    oldstat = os.fstat(fh)
379                if not fields.update_atime:
380                    attr.st_atime_ns = oldstat.st_atime_ns
381                else:
382                    attr.st_mtime_ns = oldstat.st_mtime_ns
383                if fh is None:
384                    os.utime(
385                        path,  # pyright: ignore[reportPossiblyUnboundVariable]
386                        None,
387                        follow_symlinks=False,
388                        ns=(attr.st_atime_ns, attr.st_mtime_ns),
389                    )
390                else:
391                    os.utime(fh, None, ns=(attr.st_atime_ns, attr.st_mtime_ns))
392
393        except OSError as exc:
394            assert exc.errno is not None
395            raise FUSEError(exc.errno)
396
397        return await self.getattr(inode, ctx)
398
399    async def mknod(
400        self, parent_inode: InodeT, name: bytes, mode: int, rdev: int, ctx: RequestContext
401    ) -> EntryAttributes:
402        path = os.path.join(self._inode_to_path(parent_inode), fsdecode(name))
403        try:
404            os.mknod(path, mode=(mode & ~ctx.umask), device=rdev)
405            os.chown(path, ctx.uid, ctx.gid)
406        except OSError as exc:
407            assert exc.errno is not None
408            raise FUSEError(exc.errno)
409        attr = self._getattr(path=path)
410        self._add_path(attr.st_ino, path)
411        return attr
412
413    async def mkdir(
414        self, parent_inode: InodeT, name: bytes, mode: int, ctx: RequestContext
415    ) -> EntryAttributes:
416        path = os.path.join(self._inode_to_path(parent_inode), fsdecode(name))
417        try:
418            os.mkdir(path, mode=(mode & ~ctx.umask))
419            os.chown(path, ctx.uid, ctx.gid)
420        except OSError as exc:
421            assert exc.errno is not None
422            raise FUSEError(exc.errno)
423        attr = self._getattr(path=path)
424        self._add_path(attr.st_ino, path)
425        return attr
426
427    async def statfs(self, ctx: RequestContext) -> StatvfsData:
428        root = self._inode_path_map[pyfuse3.ROOT_INODE]
429        assert isinstance(root, str)
430        stat_ = StatvfsData()
431        try:
432            statfs = os.statvfs(root)
433        except OSError as exc:
434            assert exc.errno is not None
435            raise FUSEError(exc.errno)
436        for attr in (
437            'f_bsize',
438            'f_frsize',
439            'f_blocks',
440            'f_bfree',
441            'f_bavail',
442            'f_files',
443            'f_ffree',
444            'f_favail',
445        ):
446            setattr(stat_, attr, getattr(statfs, attr))
447        stat_.f_namemax = statfs.f_namemax - (len(root) + 1)
448        return stat_
449
450    async def open(self, inode: InodeT, flags: int, ctx: RequestContext) -> FileInfo:
451        if inode in self._inode_fd_map:
452            fd = self._inode_fd_map[inode]
453            self._fd_open_count[fd] += 1
454            return FileInfo(fh=FileHandleT(fd))
455        assert flags & os.O_CREAT == 0
456        try:
457            fd = os.open(self._inode_to_path(inode), flags)
458        except OSError as exc:
459            assert exc.errno is not None
460            raise FUSEError(exc.errno)
461        self._inode_fd_map[inode] = fd
462        self._fd_inode_map[fd] = inode
463        self._fd_open_count[fd] = 1
464        return FileInfo(fh=cast(FileHandleT, fd))
465
466    async def create(
467        self, parent_inode: InodeT, name: bytes, mode: int, flags: int, ctx: RequestContext
468    ) -> tuple[FileInfo, EntryAttributes]:
469        path = os.path.join(self._inode_to_path(parent_inode), fsdecode(name))
470        try:
471            fd = os.open(path, flags | os.O_CREAT | os.O_TRUNC)
472        except OSError as exc:
473            assert exc.errno is not None
474            raise FUSEError(exc.errno)
475        attr = self._getattr(fd=fd)
476        self._add_path(attr.st_ino, path)
477        self._inode_fd_map[attr.st_ino] = fd
478        self._fd_inode_map[fd] = attr.st_ino
479        self._fd_open_count[fd] = 1
480        return (FileInfo(fh=cast(FileHandleT, fd)), attr)
481
482    async def read(self, fh: FileHandleT, off: int, size: int) -> bytes:
483        os.lseek(fh, off, os.SEEK_SET)
484        return os.read(fh, size)
485
486    async def write(self, fh: FileHandleT, off: int, buf: bytes) -> int:
487        os.lseek(fh, off, os.SEEK_SET)
488        return os.write(fh, buf)
489
490    async def release(self, fh: FileHandleT) -> None:
491        if self._fd_open_count[fh] > 1:
492            self._fd_open_count[fh] -= 1
493            return
494
495        del self._fd_open_count[fh]
496        inode = self._fd_inode_map[fh]
497        del self._inode_fd_map[inode]
498        del self._fd_inode_map[fh]
499        try:
500            os.close(fh)
501        except OSError as exc:
502            assert exc.errno is not None
503            raise FUSEError(exc.errno)
504
505
506def init_logging(debug: bool = False) -> None:
507    formatter = logging.Formatter(
508        '%(asctime)s.%(msecs)03d %(threadName)s: [%(name)s] %(message)s',
509        datefmt="%Y-%m-%d %H:%M:%S",
510    )
511    handler = logging.StreamHandler()
512    handler.setFormatter(formatter)
513    root_logger = logging.getLogger()
514    if debug:
515        handler.setLevel(logging.DEBUG)
516        root_logger.setLevel(logging.DEBUG)
517    else:
518        handler.setLevel(logging.INFO)
519        root_logger.setLevel(logging.INFO)
520    root_logger.addHandler(handler)
521
522
523def parse_args(args: list[str]) -> Namespace:
524    '''Parse command line'''
525
526    parser = ArgumentParser()
527
528    parser.add_argument('source', type=str, help='Directory tree to mirror')
529    parser.add_argument('mountpoint', type=str, help='Where to mount the file system')
530    parser.add_argument(
531        '--debug', action='store_true', default=False, help='Enable debugging output'
532    )
533    parser.add_argument(
534        '--debug-fuse', action='store_true', default=False, help='Enable FUSE debugging output'
535    )
536    parser.add_argument(
537        '--enable-writeback-cache',
538        action='store_true',
539        default=False,
540        help='Enable writeback cache (default: disabled)',
541    )
542
543    return parser.parse_args(args)
544
545
546def main() -> None:
547    options = parse_args(sys.argv[1:])
548    init_logging(options.debug)
549    operations = Operations(options.source, enable_writeback_cache=options.enable_writeback_cache)
550
551    log.debug('Mounting...')
552    fuse_options = set(pyfuse3.default_options)
553    fuse_options.add('fsname=passthroughfs')
554    if options.debug_fuse:
555        fuse_options.add('debug')
556    pyfuse3.init(operations, options.mountpoint, fuse_options)
557
558    try:
559        log.debug('Entering main loop..')
560        trio.run(pyfuse3.main)
561    except:
562        pyfuse3.close(unmount=False)
563        raise
564
565    log.debug('Unmounting..')
566    pyfuse3.close()
567
568
569if __name__ == '__main__':
570    main()