Add read-only support for zipped Dirfiles.

Separate from the Dirfile encoding scheme, GetData will read Dirfiles contained
in uncompressed Zip files. This functionality is meant for reading archival
data, so writing to these Zip files is not supported. Using the Info-ZIP `zip`
utility, a Zip file can be created by running `zip -r0 ../dirfile.zip *` from
within the root of an existing Dirfile. All encoding schemes are supported by
this functionality except for the two encoding schemes that already use Zip
files, *zzip* and *zzslim*. The encoding scheme must be specified using the
/ENCODING directive, even if the Dirfile is unencoded. For /INCLUDE directives
and LINTERP field look up table files, only relative paths are supported and
only without `./` and `../` syntax.

Although Zip files are most commonly created using _Deflate_ compression, the
Zip standard (ISO/IEC 21320-1) also supports _Store_ compression, i.e., no
compression at all. GetData's Zip file support requires _Store_ compression for
all data files, although either _Store_ compression or _Deflate_ compression
can be used for any *format* files or any LINTERP field look up table files.
With _Store_ compression, a Zip file effectively concatenates a Dirfile's
individual files together into a single file. Since a Zip file contains an
offset table, unlike a tarball, random reads are supported without the
need to load the entire file from disk.

Author: mpetroff

src/Makefile.am

@@ -80,6 +80,9 @@ MODULE_LIBS=libgetdata.la
 if USE_MODULES
 EXPORT_DYNAMIC=-export-dynamic
 DGETDATA_MODULEDIR=-DGETDATA_MODULEDIR="\"$(moduledir)\""
+EXTERNAL_CPPFLAGS=$(ZZIP_CPPFLAGS)
+EXTERNAL_LDFLAGS=$(ZZIP_LDFLAGS)
+EXTERNAL_LIBS=$(ZZIP_LIBS)
 else
 EXTERNAL_CPPFLAGS=$(SLIM_CPPFLAGS) $(GZIP_CPPFLAGS) $(BZIP2_CPPFLAGS) \
 									$(LZMA_CPPFLAGS) $(ZZIP_CPPFLAGS) $(FLAC_CPPFLAGS)

src/ascii.c

@@ -28,7 +28,7 @@ int _GD_AsciiOpen(int fd, struct gd_raw_file_* file, gd_type_t type gd_unused_,
   dtrace("%i, %p, <unused>, <unused>, %u", fd, file, mode);
 
   if (!(mode & GD_FILE_TEMP))
-    file->idata = gd_OpenAt(file->D, fd, file->name, ((mode & GD_FILE_WRITE)
+    file->idata = gd_openat_wrapper(file->D, fd, file->name, ((mode & GD_FILE_WRITE)
           ? (O_RDWR | O_CREAT) : O_RDONLY) | O_BINARY, 0666);
   else
     file->idata = _GD_MakeTempFile(file->D, fd, file->name);
@@ -49,6 +49,7 @@ int _GD_AsciiOpen(int fd, struct gd_raw_file_* file, gd_type_t type gd_unused_,
 
   file->mode = mode | GD_FILE_READ;
   file->pos = 0;
+  file->start_offset = ftello64(file->edata);
   dreturn("%i", 0);
   return 0;
 }
@@ -61,7 +62,7 @@ off64_t _GD_AsciiSeek(struct gd_raw_file_* file, off64_t count,
   dtrace("%p, %" PRId64 ", <unused>, 0x%X", file, (int64_t)count, mode);
 
   if (count < file->pos) {
-    rewind((FILE *)file->edata);
+    fseeko64((FILE *)file->edata, file->start_offset, SEEK_SET); /* rewind */
     file->pos = 0;
   }
 
@@ -285,7 +286,7 @@ off64_t _GD_AsciiSize(int dirfd, struct gd_raw_file_* file,
 
   dtrace("%i, %p, <unused>, <unused>", dirfd, file);
 
-  fd = gd_OpenAt(file->D, dirfd, file->name, O_RDONLY, 0666);
+  fd = gd_openat_wrapper(file->D, dirfd, file->name, O_RDONLY, 0666);
   if (fd < 0) {
     dreturn("%i", -1);
     return -1;

src/bzip.c

@@ -56,7 +56,7 @@ static struct gd_bzdata *_GD_Bzip2DoOpen(int dirfd, struct gd_raw_file_* file,
   file->error = BZ_IO_ERROR;
 
   if (mode & GD_FILE_READ) {
-    fd = gd_OpenAt(file->D, dirfd, file->name, O_RDONLY | O_BINARY, 0666);
+    fd = gd_openat_wrapper(file->D, dirfd, file->name, O_RDONLY | O_BINARY, 0666);
   } else if (mode & GD_FILE_TEMP) {
     fd = _GD_MakeTempFile(file->D, dirfd, file->name);
     fdmode = "wb";

src/close.c

@@ -76,6 +76,11 @@ static void _GD_FreeD(DIRFILE *D, int keep_dirfile)
   }
   free(D->dir);
 
+#ifdef HAVE_ZZIP_LIB_H
+  if (D->zzip_dir)
+    zzip_dir_close(D->zzip_dir);
+#endif
+
   if (!keep_dirfile)
     free(D);
 

src/common.c

@@ -346,7 +346,7 @@ static int lutcmp(const void* a, const void* b)
 */
 int _GD_ReadLinterpFile(DIRFILE *restrict D, gd_entry_t *restrict E)
 {
-  FILE *fp;
+  FILE *fp = NULL;
   struct gd_lut_ *ptr;
   int i, fd;
   int dir = -1;
@@ -364,15 +364,23 @@ int _GD_ReadLinterpFile(DIRFILE *restrict D, gd_entry_t *restrict E)
       return 1;
     }
 
-  fd = gd_OpenAt(D, E->e->u.linterp.table_dirfd, E->e->u.linterp.table_file,
-      O_RDONLY, 0666);
-  if (fd == -1) {
-    _GD_SetError(D, GD_E_IO, GD_E_IO_OPEN, E->EN(linterp,table), 0, NULL);
-    dreturn("%i", 1);
-    return 1;
+  if (!D->zzip_dir) {
+    fd = gd_openat_wrapper(D, E->e->u.linterp.table_dirfd, E->e->u.linterp.table_file,
+        O_RDONLY, 0666);
+    if (fd == -1) {
+      _GD_SetError(D, GD_E_IO, GD_E_IO_OPEN, E->EN(linterp,table), 0, NULL);
+      dreturn("%i", 1);
+      return 1;
+    }
+    fp = fdopen(fd, "rb");
+  } else {
+    if (!gd_zip_read_file(D, E->e->u.linterp.table_dirfd, E->e->u.linterp.table_file, &fp)) {
+      _GD_SetError(D, GD_E_IO, GD_E_IO_OPEN, E->EN(linterp,table), 0, NULL);
+      dreturn("%i", 1);
+      return 1;
+    }
   }
 
-  fp = fdopen(fd, "rb");
   if (fp == NULL) {
     _GD_SetError(D, GD_E_IO, GD_E_IO_OPEN, E->EN(linterp,table), 0, NULL);
     dreturn("%i", 1);
@@ -1430,9 +1438,13 @@ int _GD_GrabDir(DIRFILE *D, int dirfd, const char *name, int canonical)
   free(path);
 
   if (D->dir[D->ndir].fd == -1) {
-    free(D->dir[D->ndir].path);
-    dreturn("%i", -1);
-    return -1;
+    if (D->zzip_dir) {
+      D->dir[D->ndir].fd = dup(dirfd);
+    } else {
+      free(D->dir[D->ndir].path);
+      dreturn("%i", -1);
+      return -1;
+    }
   }
 #endif
   D->ndir++;

src/flac.c

@@ -151,7 +151,7 @@ static struct gd_flacdata *_GD_FlacDoOpen(int dirfd, struct gd_raw_file_* file,
   dtrace("%i, %p, 0x%X, %i, 0x%X", dirfd, file, data_type, swap, mode);
 
   if (mode & GD_FILE_READ) {
-    fd = gd_OpenAt(file->D, dirfd, file->name, O_RDONLY | O_BINARY, 0666);
+    fd = gd_openat_wrapper(file->D, dirfd, file->name, O_RDONLY | O_BINARY, 0666);
   } else if (mode & GD_FILE_TEMP) {
     fd = _GD_MakeTempFile(file->D, dirfd, file->name);
     fdmode = "wb";

src/gzip.c

@@ -46,7 +46,7 @@ int _GD_GzipOpen(int fd, struct gd_raw_file_* file,
   dtrace("%i, %p, <unused>, <unused>, 0x%X", fd, file, mode);
 
   if (mode & GD_FILE_READ) {
-    file->idata = gd_OpenAt(file->D, fd, file->name, O_RDONLY | O_BINARY, 0666);
+    file->idata = gd_openat_wrapper(file->D, fd, file->name, O_RDONLY | O_BINARY, 0666);
     gzmode = "r";
   } else if (mode & GD_FILE_TEMP) {
     file->idata = _GD_MakeTempFile(file->D, fd, file->name);
@@ -185,17 +185,32 @@ off64_t _GD_GzipSize(int dirfd, struct gd_raw_file_ *file, gd_type_t data_type,
 
   dtrace("%i, %p, 0x%X, <unused>", dirfd, file, data_type);
 
-  fd = gd_OpenAt(file->D, dirfd, file->name, O_RDONLY | O_BINARY, 0666);
+  fd = gd_openat_wrapper(file->D, dirfd, file->name, O_RDONLY | O_BINARY, 0666);
   if (fd < 0) {
     dreturn("%i", -1);
     return -1;
   }
 
   /* seek to the end */
-  if (lseek64(fd, -4, SEEK_END) == -1) {
-    dreturn("%i", -1);
-    return -1;
+#ifdef HAVE_ZZIP_LIB_H
+  if (file->D->zzip_dir) {
+    ZZIP_FILE *zzip_file = zzip_file_open(file->D->zzip_dir, file->name, O_RDONLY | O_BINARY);
+    if (zzip_file && zzip_file->method == 0) {
+      lseek64(dirfd, zzip_file->dataoffset + zzip_file->csize - 4, SEEK_SET);
+      zzip_file_close(zzip_file);
+    } else {
+      dreturn("%i", -1);
+      return -1;
+    }
+  } else {
+#endif
+    if (lseek64(fd, -4, SEEK_END) == -1) {
+      dreturn("%i", -1);
+      return -1;
+    }
+#ifdef HAVE_ZZIP_LIB_H
   }
+#endif
   if (read(fd, &size, 4) < 4) {
     dreturn("%i", -1);
     return -1;

src/include.c

@@ -253,14 +253,14 @@ int _GD_Include(DIRFILE *D, struct parser_state *p, const char *ename,
 
   /* Open the containing directory */
   dirfd = _GD_GrabDir(D, D->fragment[parent].dirfd, temp_buf1, 1);
-  if (dirfd == -1 && D->error == GD_E_OK)
+  if (dirfd == -1 && D->error == GD_E_OK && !D->zzip_dir)
     _GD_SetError(D, GD_E_IO, GD_E_IO_INCL, p->file, p->line, ename);
   if (D->error)
     goto include_error;
 
   /* Reject weird stuff */
   if (gd_StatAt(D, dirfd, base, &statbuf, 0)) {
-    if (!(p->flags & GD_CREAT)) {
+    if (!(p->flags & GD_CREAT) && !D->zzip_dir) {
       _GD_SetError(D, GD_E_IO, GD_E_IO_INCL, p->file, p->line, temp_buf1);
       _GD_ReleaseDir(D, dirfd);
       goto include_error;
@@ -280,19 +280,33 @@ int _GD_Include(DIRFILE *D, struct parser_state *p, const char *ename,
   }
 
   /* Try to open the file */
-  i = gd_OpenAt(D, dirfd, base,
-      ((p->flags & (GD_CREAT | GD_TRUNC)) ? O_RDWR : O_RDONLY) |
-      ((p->flags & GD_CREAT) ? O_CREAT : 0) |
-      ((p->flags & GD_TRUNC) ? O_TRUNC : 0) |
-      ((p->flags & GD_EXCL) ? O_EXCL : 0) | O_BINARY, 0666);
+  if (D->zzip_dir) {
+    i = open(D->name, O_RDONLY | O_BINARY);
+  } else {
+    i = gd_OpenAt(D, dirfd, base,
+        ((p->flags & (GD_CREAT | GD_TRUNC)) ? O_RDWR : O_RDONLY) |
+        ((p->flags & GD_CREAT) ? O_CREAT : 0) |
+        ((p->flags & GD_TRUNC) ? O_TRUNC : 0) |
+        ((p->flags & GD_EXCL) ? O_EXCL : 0) | O_BINARY, 0666);
+  }
 
   if (i < 0) {
     _GD_SetError(D, GD_E_IO, GD_E_IO_INCL, p->file, p->line, temp_buf1);
     _GD_ReleaseDir(D, dirfd);
     goto include_error;
   }
 
-  new_fp = fdopen(i, (p->flags & (GD_CREAT | GD_TRUNC)) ? "rb+" : "rb");
+  if (!D->zzip_dir) {
+    new_fp = fdopen(i, (p->flags & (GD_CREAT | GD_TRUNC)) ? "rb+" : "rb");
+  } else {
+    if (!gd_zip_read_file(D, dirfd, base, &new_fp)) {
+      _GD_SetError(D, GD_E_IO, GD_E_IO_INCL, p->file, p->line, temp_buf1);
+      _GD_ReleaseDir(D, dirfd);
+      close(i);
+      goto include_error;
+    }
+    close(i);
+  }
 
   /* If opening the file failed, set the error code and abort parsing. */
   if (new_fp == NULL) {

src/internal.h

@@ -113,6 +113,10 @@
 #include <regex.h>
 #endif
 
+#ifdef HAVE_ZZIP_LIB_H
+#include <zzip/lib.h>
+#include <zzip/file.h>
+#endif
 
 /* MSCVRT defines size_t but not ssize_t */
 #ifdef __MSVCRT__
@@ -994,6 +998,7 @@ struct gd_raw_file_ {
   DIRFILE *D;
   unsigned int mode;
   off64_t pos;
+  off64_t start_offset;
 };
 
 /* linterp table datum */
@@ -1283,6 +1288,13 @@ struct gd_dirfile_ {
   /* syntax error callback */
   gd_parser_callback_t sehandler;
   void* sehandler_extra;
+
+  /* for zipped Dirfile support */
+#ifdef HAVE_ZZIP_LIB_H
+  ZZIP_DIR *zzip_dir;
+#else
+  void *zzip_dir;
+#endif
 };
 
 /* The caller's preferred memory manager */
@@ -1434,6 +1446,9 @@ gd_entry_t *_GD_ParseFieldSpec(DIRFILE *restrict,
 char *_GD_ParseFragment(FILE *restrict, DIRFILE*, struct parser_state *restrict,
     int, int);
 void _GD_PerformRename(DIRFILE *restrict, struct gd_rename_data_ *restrict);
+int gd_openat_wrapper(const DIRFILE *D, int dirfd, const char *name, int flags,
+    mode_t mode);
+int gd_zip_read_file(const DIRFILE *D, int dirfd, const char *name, FILE **fp);
 struct gd_rename_data_ *_GD_PrepareRename(DIRFILE *restrict, char *restrict,
     size_t, gd_entry_t *restrict, int, unsigned);
 int _GD_ReadLinterpFile(DIRFILE *restrict, gd_entry_t *restrict);

src/iopos.c

@@ -415,5 +415,112 @@ off_t gd_seek(DIRFILE *D, const char *field_code, off_t frame_num,
 {
   return (off_t)gd_seek64(D, field_code, frame_num, sample_num, whence);
 }
+
+int gd_openat_wrapper(const DIRFILE *D, int dirfd, const char *name, int flags,
+    mode_t mode)
+{
+  int ret;
+  const char *dir = NULL;
+  const char *p1, *p2, *p;
+  char *newp;
+#ifdef HAVE_ZZIP_LIB_H
+  ZZIP_FILE *zzip_file;
+#endif
+
+  dtrace("%p, %i, \"%s\", 0x%X, 0%o", D, dirfd, name, flags, mode);
+
+#ifdef HAVE_ZZIP_LIB_H
+  if (!D->zzip_dir) {
+#endif
+    ret = gd_OpenAt(D, dirfd, name, flags, mode);
+#ifdef HAVE_ZZIP_LIB_H
+  } else {
+    dir = _GD_DirName(D, dirfd);
+    /* find where zip file and desired file's absolute paths differ */
+    for (p1 = dir, p2 = D->dir[0].path; *p1 && *p1 == *p2; p1++, p2++);
+    p = *(p1) && *(p1+1) ? p1+1 : NULL;
+    if (p) {
+      /* if paths differ (i.e. not root of zip), construct relative path */
+      newp = malloc(strlen(p) + 1 + strlen(name) + 1);
+      strcpy(newp, p);
+      newp[strlen(p)] = '/';
+      strcpy(newp + strlen(p) + 1, name);
+      zzip_file = zzip_file_open(D->zzip_dir, newp, flags | O_BINARY);
+      free(newp);
+    } else {
+      /* root of zip */
+      zzip_file = zzip_file_open(D->zzip_dir, name, flags | O_BINARY);
+    }
+    if (zzip_file && zzip_file->method == 0) {
+      lseek64(dirfd, zzip_file->dataoffset, SEEK_SET);
+      zzip_file_close(zzip_file);
+      ret = dup(dirfd);
+    } else {
+      ret = EACCES;
+    }
+  }
+#endif
+
+  dreturn("%i", ret);
+  return ret;
+}
+
+int gd_zip_read_file(const DIRFILE *D, int dirfd, const char *name, FILE **fp)
+{
+  int ret;
+  const char *p1, *p2, *p;
+  char *newp;
+#ifdef HAVE_ZZIP_LIB_H
+  ZZIP_FILE *zzip_file;
+#endif
+  const char *dir = NULL;
+
+  dtrace("%p, %i, \"%s\", %p", D, dirfd, name, fp);
+
+#ifdef HAVE_ZZIP_LIB_H
+  dir = _GD_DirName(D, dirfd);
+  /* find where zip file and desired file's absolute paths differ */
+  if (D->ndir > 0) {
+    for (p1 = dir, p2 = D->dir[0].path; *p1 && *p1 == *p2; p1++, p2++);
+    p = *(p1) && *(p1+1) ? p1+1 : NULL;
+  } else {
+    p = NULL;
+  }
+  if (p) {
+    /* if paths differ (i.e. not root of zip), construct relative path */
+    newp = malloc(strlen(p) + 1 + strlen(name) + 1);
+    strcpy(newp, p);
+    newp[strlen(p)] = '/';
+    strcpy(newp + strlen(p) + 1, name);
+    zzip_file = zzip_file_open(D->zzip_dir, newp, O_RDONLY | O_BINARY);
+    free(newp);
+  } else {
+    /* root of zip */
+    zzip_file = zzip_file_open(D->zzip_dir, name, O_RDONLY | O_BINARY);
+  }
+  if (zzip_file /*&& zzip_file->method == 0*/) {
+    char *contents = malloc(zzip_file->usize + 1);
+    if (!zzip_file_read(zzip_file, contents, zzip_file->usize)) {
+      dreturn("%i", 0);
+      return 0;
+    }
+    contents[zzip_file->usize] = 0;
+    *fp = fmemopen(NULL, zzip_file->usize + 1, "r+");
+    fwrite(contents, 1, zzip_file->usize + 1, *fp);
+    rewind(*fp);
+    free(contents);
+    zzip_file_close(zzip_file);
+    ret = 1;
+  } else {
+    ret = 0;
+  }
+
+  dreturn("%i", ret);
+  return ret;
+#else
+  dreturn("%i", 0);
+  return 0;
+#endif
+}
 /* vim: ts=2 sw=2 et tw=80
 */