Merge pull request #84 from worldcoin/ewoolsey/file-create

fix for mmap file creation

.gitignore

@@ -4,3 +4,4 @@ Cargo.lock
 snarkfiles_tmp
 semaphore_files
 .idea
+lcov.info

src/cascading_merkle_tree.rs

@@ -347,6 +347,11 @@ where
     /// Validates all elements of the storage, ensuring that they
     /// correspond to a valid tree.
     pub fn validate(&self) -> Result<()> {
+        debug_assert_eq!(
+            self.root,
+            self.compute_from_storage_tip(0),
+            "Root hash does not match recomputed root hash"
+        );
         self.storage.validate(&self.empty_value)
     }
 
@@ -985,6 +990,23 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_extend_from_slice_2() {
+        for increment in 1..20 {
+            let mut tree = CascadingMerkleTree::<PoseidonHash>::new(vec![], 30, &Field::ZERO);
+            let mut vec = vec![];
+            for _ in 0..20 {
+                let slice = (0..increment)
+                    .map(|_| Field::from(rand::random::<usize>()))
+                    .collect::<Vec<_>>();
+                tree.extend_from_slice(&slice);
+                vec.extend_from_slice(&slice);
+                tree.validate().unwrap();
+                assert_eq!(tree.leaves().collect::<Vec<_>>(), vec);
+            }
+        }
+    }
+
     #[test]
     fn test_vec_realloc_speed() {
         let empty = 0;

src/generic_storage/mmap_vec.rs

@@ -46,9 +46,8 @@ impl<T: Pod> MmapVec<T> {
     /// Notably this means that there can exist no other mutable mappings to the
     /// same file in this process or any other
     pub unsafe fn create(file: File) -> color_eyre::Result<Self> {
-        let initial_byte_len = META_SIZE;
-
-        file.set_len(initial_byte_len as u64)
+        file.set_len(0)?;
+        file.set_len(META_SIZE as u64)
             .context("Failed to resize underlying file")?;
 
         let mut s = Self::restore(file)?;
@@ -63,7 +62,12 @@ impl<T: Pod> MmapVec<T> {
     /// # Safety
     /// Same requirements as `restore`
     pub unsafe fn restore_from_path(file_path: impl AsRef<Path>) -> color_eyre::Result<Self> {
-        let file = OpenOptions::new().read(true).write(true).open(file_path)?;
+        let file = OpenOptions::new()
+            .read(true)
+            .write(true)
+            .create(true)
+            .truncate(false)
+            .open(file_path)?;
 
         Self::restore(file)
     }
@@ -245,6 +249,202 @@ mod tests {
 
     use super::*;
 
+    #[test]
+    #[allow(clippy::manual_bits)]
+    fn test_capacity_push() {
+        let f = tempfile::NamedTempFile::new().unwrap();
+        let file_path = f.path().to_owned();
+
+        let mut storage: MmapVec<u32> = unsafe { MmapVec::create(f.reopen().unwrap()).unwrap() };
+        assert_eq!(storage.capacity, 0);
+        assert_eq!(
+            std::fs::metadata(&file_path).unwrap().len(),
+            META_SIZE as u64
+        );
+
+        storage.push(0);
+        assert_eq!(storage.capacity, 1);
+        assert_eq!(
+            std::fs::metadata(&file_path).unwrap().len() as usize,
+            size_of::<u32>() + META_SIZE
+        );
+
+        storage.push(0);
+        assert_eq!(storage.capacity, 2);
+        assert_eq!(
+            std::fs::metadata(&file_path).unwrap().len() as usize,
+            size_of::<u32>() * 2 + META_SIZE
+        );
+
+        storage.push(0);
+        assert_eq!(storage.capacity, 4);
+        assert_eq!(
+            std::fs::metadata(&file_path).unwrap().len() as usize,
+            size_of::<u32>() * 4 + META_SIZE
+        );
+
+        storage.push(0);
+        assert_eq!(storage.capacity, 4);
+        assert_eq!(
+            std::fs::metadata(&file_path).unwrap().len() as usize,
+            size_of::<u32>() * 4 + META_SIZE
+        );
+
+        storage.push(0);
+        assert_eq!(storage.capacity, 8);
+        assert_eq!(
+            std::fs::metadata(&file_path).unwrap().len() as usize,
+            size_of::<u32>() * 8 + META_SIZE
+        );
+    }
+
+    #[test]
+    #[allow(clippy::manual_bits)]
+    fn test_capacity_extend() {
+        let f = tempfile::NamedTempFile::new().unwrap();
+        let file_path = f.path().to_owned();
+
+        let mut storage: MmapVec<u32> = unsafe { MmapVec::create(f.reopen().unwrap()).unwrap() };
+        assert_eq!(storage.capacity, 0);
+        assert_eq!(
+            std::fs::metadata(&file_path).unwrap().len(),
+            META_SIZE as u64
+        );
+
+        storage.extend_from_slice(&[0, 0]);
+        assert_eq!(storage.capacity, 2);
+        assert_eq!(
+            std::fs::metadata(&file_path).unwrap().len() as usize,
+            size_of::<u32>() * 2 + META_SIZE
+        );
+
+        storage.extend_from_slice(&[0, 0, 0]);
+        assert_eq!(storage.capacity, 8);
+        assert_eq!(
+            std::fs::metadata(&file_path).unwrap().len() as usize,
+            size_of::<u32>() * 8 + META_SIZE
+        );
+
+        storage.extend_from_slice(&[0]);
+        assert_eq!(storage.capacity, 8);
+        assert_eq!(
+            std::fs::metadata(&file_path).unwrap().len() as usize,
+            size_of::<u32>() * 8 + META_SIZE
+        );
+    }
+
+    #[test]
+    #[allow(clippy::manual_bits)]
+    fn test_capacity_create() {
+        let f = tempfile::NamedTempFile::new().unwrap();
+        let file_path = f.path().to_owned();
+
+        let mut storage: MmapVec<u32> = unsafe { MmapVec::create(f.reopen().unwrap()).unwrap() };
+        assert_eq!(storage.capacity, 0);
+        assert_eq!(
+            std::fs::metadata(&file_path).unwrap().len() as usize,
+            META_SIZE
+        );
+
+        storage.extend_from_slice(&[0, 0, 0, 0, 0]);
+        assert_eq!(storage.capacity, 8);
+        assert_eq!(
+            std::fs::metadata(&file_path).unwrap().len() as usize,
+            size_of::<u32>() * 8 + META_SIZE
+        );
+
+        let storage: MmapVec<u32> = unsafe { MmapVec::create(f.reopen().unwrap()).unwrap() };
+        assert_eq!(storage.capacity, 0);
+        assert_eq!(
+            std::fs::metadata(&file_path).unwrap().len() as usize,
+            META_SIZE
+        );
+    }
+
+    #[test]
+    #[allow(clippy::manual_bits)]
+    fn test_capacity_create_from_path() {
+        let f = tempfile::NamedTempFile::new().unwrap();
+        let file_path = f.path().to_owned();
+
+        let mut storage: MmapVec<u32> = unsafe { MmapVec::create(f.reopen().unwrap()).unwrap() };
+        assert_eq!(storage.capacity, 0);
+        assert_eq!(
+            std::fs::metadata(&file_path).unwrap().len() as usize,
+            META_SIZE
+        );
+
+        storage.extend_from_slice(&[0, 0, 0, 0, 0]);
+        assert_eq!(storage.capacity, 8);
+        assert_eq!(
+            std::fs::metadata(&file_path).unwrap().len() as usize,
+            size_of::<u32>() * 8 + META_SIZE
+        );
+
+        let storage: MmapVec<u32> = unsafe { MmapVec::create_from_path(&file_path).unwrap() };
+        assert_eq!(storage.capacity, 0);
+        assert_eq!(
+            std::fs::metadata(&file_path).unwrap().len() as usize,
+            META_SIZE
+        );
+    }
+
+    #[test]
+    #[allow(clippy::manual_bits)]
+    fn test_capacity_restore() {
+        let f = tempfile::NamedTempFile::new().unwrap();
+        let file_path = f.path().to_owned();
+
+        let mut storage: MmapVec<u32> = unsafe { MmapVec::create(f.reopen().unwrap()).unwrap() };
+        assert_eq!(storage.capacity, 0);
+        assert_eq!(
+            std::fs::metadata(&file_path).unwrap().len() as usize,
+            META_SIZE
+        );
+
+        storage.extend_from_slice(&[0, 0, 0, 0, 0]);
+        assert_eq!(storage.capacity, 8);
+        assert_eq!(
+            std::fs::metadata(&file_path).unwrap().len() as usize,
+            size_of::<u32>() * 8 + META_SIZE
+        );
+
+        let storage: MmapVec<u32> = unsafe { MmapVec::restore(f.reopen().unwrap()).unwrap() };
+        assert_eq!(storage.capacity, 8);
+        assert_eq!(
+            std::fs::metadata(&file_path).unwrap().len() as usize,
+            size_of::<u32>() * 8 + META_SIZE
+        );
+    }
+
+    #[test]
+    #[allow(clippy::manual_bits)]
+    fn test_capacity_restore_from_path() {
+        let f = tempfile::NamedTempFile::new().unwrap();
+        let file_path = f.path().to_owned();
+
+        let mut storage: MmapVec<u32> = unsafe { MmapVec::create(f.reopen().unwrap()).unwrap() };
+        assert_eq!(storage.capacity, 0);
+        assert_eq!(
+            std::fs::metadata(&file_path).unwrap().len() as usize,
+            META_SIZE
+        );
+
+        storage.extend_from_slice(&[0, 0, 0, 0, 0]);
+        assert_eq!(storage.capacity, 8);
+        assert_eq!(
+            std::fs::metadata(&file_path).unwrap().len() as usize,
+            size_of::<u32>() * 8 + META_SIZE
+        );
+
+        let storage: MmapVec<u32> = unsafe { MmapVec::restore_from_path(&file_path).unwrap() };
+        assert_eq!(storage.capacity, 8);
+        assert_eq!(
+            std::fs::metadata(&file_path).unwrap().len() as usize,
+            size_of::<u32>() * 8 + META_SIZE
+        );
+    }
+
     #[test]
     fn test_mmap_vec() {
         let f = tempfile::tempfile().unwrap();