WIP pdf extractor

Author: joepio

Cargo.lock

@@ -1,5 +1,6 @@
 [workspace]
 members = [
+  "atomizer",
   "server",
   "cli",
   "lib",

Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+description = "Turn files into Atomic Data."
+edition = "2021"
+name = "atomizer"
+version = "0.1.0"
+
+[dependencies]
+atomic_lib = {version = "0.34.0", path = "../lib"}
+# Should match the version of pdf-extract
+lopdf = "0.26"
+mime_guess = "2.0.4"
+pdf-extract = {git = "https://github.com/Hessesian/pdf-extract"}

atomizer/Cargo.toml

@@ -0,0 +1,38 @@
+use std::{collections::HashMap, error::Error};
+
+use atomic_lib::resources::PropVals;
+
+use crate::pdf;
+
+pub struct File {
+    filename: String,
+    mime: String,
+    bytes: Vec<u8>,
+}
+
+impl File {
+    pub fn open(filename: &str) -> Result<File, Box<dyn Error>> {
+        let bytes = std::fs::read(filename)?;
+        let mime = mime_guess::from_path(filename)
+            .first_or_octet_stream()
+            .to_string();
+
+        Ok(File {
+            filename: filename.to_string(),
+            mime,
+            bytes,
+        })
+    }
+
+    /// Transforms an input file into an Atomic Data [Resource]
+    pub fn atomize(&self) -> PropVals {
+        match self.mime.as_str() {
+            "application/pdf" => pdf::atomize(self),
+            _ => HashMap::new(),
+        }
+    }
+
+    pub fn bytes(&self) -> &[u8] {
+        &self.bytes
+    }
+}

atomizer/src/file.rs

@@ -0,0 +1,4 @@
+mod file;
+mod pdf;
+
+use atomic_lib::Resource;

atomizer/src/lib.rs

@@ -0,0 +1,27 @@
+use atomic_lib::resources::PropVals;
+use lopdf::content;
+
+const content_prop: &str = "content";
+
+pub fn atomize(file: &crate::file::File) -> PropVals {
+    let mut props = PropVals::new();
+    let mut s = String::new();
+    let mut output = pdf_extract::PlainTextOutput::new(&mut s);
+    let text = pdf_extract::extract_text_mem(file.bytes()).unwrap();
+    props.insert(content_prop.into(), atomic_lib::Value::String(text));
+    props
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::file::File;
+
+    #[test]
+    fn load_pdf() {
+        let f = File::open("./test/docs-demo.pdf").unwrap();
+        let propvals = f.atomize();
+        let content = propvals.get(content_prop).unwrap();
+        assert!(content.to_string().contains("Atomic Data"));
+    }
+}

atomizer/src/pdf.rs

@@ -9,7 +9,7 @@ use crate::{
     parse::ParseOpts,
     schema::{Class, Property},
     storelike::Query,
-    urls, Resource, Storelike, Value,
+    urls, Storelike, Value,
 };
 
 /// Populates a store with some of the most fundamental Properties and Classes needed to bootstrap the whole.