Merge pull request #326 from korpling/fix-large-files-in-zip

Fix exporting large corpora to ZIP files

Author: thomaskrause

CHANGELOG.md

@@ -5,6 +5,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+
+- Export to ZIP would fail if the contained GraphML was too large with error
+`Error: I/O error: Large file option has not been set`. Use the ZIP64 extension
+(which should be supported in most current tools and libraries) to write the ZIP
+file.
+
 ## [3.8.1] - 2025-05-22
 
 ### Fixed

cli/tests/cli.rs

@@ -2,7 +2,7 @@ use assert_cmd::prelude::*;
 use insta::Settings;
 use insta_cmd::assert_cmd_snapshot;
 use serial_test::serial;
-use std::process::Command;
+use std::{path::Path, process::Command};
 
 fn standard_filter() -> Settings {
     let mut settings = insta::Settings::clone_current();
@@ -88,3 +88,25 @@ fn list_corpora_partially_loaded() -> Result<(), Box<dyn std::error::Error>> {
 
     Ok(())
 }
+
+#[test]
+#[serial]
+fn export_to_zip_file() -> Result<(), Box<dyn std::error::Error>> {
+    let mut cmd = Command::cargo_bin("annis")?;
+
+    cmd.arg("../graphannis/tests/data/")
+        .arg("-c")
+        .arg("corpus sample-disk-based-3.3")
+        .arg("-c")
+        .arg("export sample-disk-based-3.3.zip");
+
+    let settings = standard_filter();
+    settings.bind(|| assert_cmd_snapshot!(cmd));
+
+    // Check that the file has been created
+    let p = Path::new("sample-disk-based-3.3.zip");
+    assert_eq!(true, p.is_file());
+    // Cleanup created file
+    std::fs::remove_file(p)?;
+    Ok(())
+}

cli/tests/snapshots/cli__export_to_zip_file.snap

@@ -0,0 +1,22 @@
+---
+source: cli/tests/cli.rs
+info:
+  program: annis
+  args:
+    - "../graphannis/tests/data/"
+    - "-c"
+    - corpus sample-disk-based-3.3
+    - "-c"
+    - export sample-disk-based-3.3.zip
+---
+success: true
+exit_code: 0
+----- stdout -----
+12:00:00[INFO] Loaded corpus sample-disk-based-3.3
+12:00:00[INFO] exporting all available annotation keys
+12:00:00[INFO] exporting nodes
+12:00:00[INFO] exporting edges
+12:00:00[INFO] exported corpora ["sample-disk-based-3.3"] in 0s
+graphANNIS says good-bye!
+
+----- stderr -----

graphannis/src/annis/db/corpusstorage.rs

@@ -1245,8 +1245,9 @@ impl CorpusStorage {
         W: Write + Seek,
         F: Fn(&str),
     {
-        let options =
-            zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Deflated);
+        let options = zip::write::FileOptions::default()
+            .compression_method(zip::CompressionMethod::Deflated)
+            .large_file(true);
 
         let mut base_path = PathBuf::default();
         if use_corpus_subdirectory {