Feature/pagination (#27)

- Gradle Dependency Management Update. Now requires Gradle 7.5
- Several Dependencies Versions upgraded, including MongoDB
- Breaking Change in DB of document api: Introduction of bucket pattern
- Breaking Change in QueryPiD: Result Format changed to include pagination information
- New CH version number
- Integration tests aligned with pagination
- Default sort set to descending

clearing-house-app/core-lib/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "core-lib"
-version = "0.8.0"
+version = "0.9.0"
 authors = [
     "Mark Gall <mark.gall@aisec.fraunhofer.de>",
     "Georg Bramm <georg.bramm@aisec.fraunhofer.de>",
@@ -21,7 +21,7 @@ figment = { version = "0.10", features = ["yaml", "env"] }
 generic-array = "0.14.4"
 hex = "0.4.2"
 log = "0.4"
-mongodb ="2.0.1"
+mongodb ="2.3.0"
 num-bigint = "0.4.3"
 openssh-keys = "0.5.0"
 percent-encoding = "2.1.0"

clearing-house-app/core-lib/src/api/auth.rs

@@ -13,7 +13,6 @@ use rocket::request::{self, Request, FromRequest};
 use serde::{Deserialize, Serialize};
 use serde::de::DeserializeOwned;
 use std::fmt::Debug;
-use biscuit::jws::Compact;
 
 use crate::{
     constants::{
@@ -23,7 +22,6 @@ use crate::{
     errors::*,
     api::client::daps_api::DapsApiClient,
 };
-use crate::api::claims::IdsClaims;
 use crate::model::JwksCache;
 
 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]

clearing-house-app/core-lib/src/api/client/document_api.rs

@@ -2,7 +2,7 @@ use reqwest::Client;
 use reqwest::StatusCode;
 use reqwest::header::{HeaderValue, CONTENT_TYPE};
 use serde_json;
-use crate::api::{ApiClient, DocumentReceipt};
+use crate::api::{ApiClient, DocumentReceipt, QueryResult};
 use crate::constants::{ROCKET_DOC_API, DOCUMENT_API_URL};
 use crate::errors::*;
 use crate::model::document::Document;
@@ -66,39 +66,33 @@ impl DocumentApiClient{
         Ok(doc)
     }
 
-    pub fn get_documents_for_pid(&self, token: &String, pid: &String) -> Result<Vec<Document>>{
+    pub fn get_documents(&self, token: &String, pid: &String, page: i32, size: i32, sort: SortingOrder, date_from: Option<String>, date_to: Option<String>) -> Result<QueryResult>{
         let document_url = format!("{}{}/{}", self.uri, ROCKET_DOC_API, url_encode(pid));
         let client = Client::new();
 
         debug!("calling {}", &document_url);
-        let mut response = client
-            .get(document_url.as_str())
-            .header(CONTENT_TYPE, HeaderValue::from_static("application/json"))
-            .bearer_auth(token)
-            .send()?;
-
-        debug!("Status Code: {}", &response.status());
-        let docs: Vec<Document> = response.json()?;
-        Ok(docs)
-    }
-
-    pub fn get_documents_for_pid_paginated(&self, token: &String, pid: &String, page: i32, size: i32, sort: SortingOrder) -> Result<Vec<Document>>{
-        let document_url = format!("{}{}/{}", self.uri, ROCKET_DOC_API, url_encode(pid));
-        let client = Client::new();
 
-        debug!("calling {}", &document_url);
-        let mut response = client
+        let mut request = client
             .get(document_url.as_str())
             .header(CONTENT_TYPE, HeaderValue::from_static("application/json"))
             .query(&[("page", page)])
             .query(&[("size", size)])
             .query(&[("sort", sort)])
-            .bearer_auth(token)
-            .send()?;
+            .bearer_auth(token);
+
+        if date_from.is_some(){
+            request = request.query(&[("date_from", date_from.unwrap())]);
+        }
+
+        if date_to.is_some(){
+            request = request.query(&[("date_to", date_to.unwrap())]);
+        }
+
+        let mut response = request.send()?;
 
         debug!("Status Code: {}", &response.status());
-        let docs: Vec<Document> = response.json()?;
-        Ok(docs)
+        let result: QueryResult = response.json()?;
+        Ok(result)
     }
 
     pub fn create_document(&self, token: &String, doc: &Document) -> Result<DocumentReceipt> {

clearing-house-app/core-lib/src/api/mod.rs

@@ -1,5 +1,6 @@
 use std::string::ToString;
 use rocket::serde::json::Value;
+use crate::model::document::Document;
 
 pub mod auth;
 pub mod claims;
@@ -51,3 +52,26 @@ impl DocumentReceipt{
         }
     }
 }
+
+#[derive(Clone, Serialize, Deserialize, Debug)]
+pub struct QueryResult{
+    pub date_from: i64,
+    pub date_to: i64,
+    pub page: Option<i32>,
+    pub size: Option<i32>,
+    pub order: String,
+    pub documents: Vec<Document>
+}
+
+impl QueryResult{
+    pub fn new(date_from: i64, date_to: i64, page: Option<i32>, size: Option<i32>, order: String, documents: Vec<Document>) -> QueryResult{
+        QueryResult{
+            date_from,
+            date_to,
+            page,
+            size,
+            order,
+            documents
+        }
+    }
+}

clearing-house-app/core-lib/src/constants.rs

@@ -36,6 +36,7 @@ pub const DOCUMENT_DB: &'static str = "document";
 pub const KEYRING_DB: &'static str = "keyring";
 pub const PROCESS_DB: &'static str = "process";
 pub const MONGO_COLL_DOCUMENTS: &'static str = "documents";
+pub const MONGO_COLL_DOCUMENT_BUCKET: &'static str = "document_bucket";
 pub const MONGO_COLL_DOC_TYPES: &'static str = "doc_types";
 pub const MONGO_COLL_DOC_PARTS: &'static str = "parts";
 pub const MONGO_COLL_PROCESSES: &'static str = "processes";
@@ -52,11 +53,18 @@ pub const MONGO_OWNER: &'static str = "owner";
 pub const MONGO_TS: &'static str = "ts";
 pub const MONGO_TC: &'static str = "tc";
 
+pub const MONGO_DOC_ARRAY: &'static str = "documents";
+pub const MONGO_COUNTER: &'static str = "counter";
+pub const MONGO_FROM_TS: &'static str = "from_ts";
+pub const MONGO_TO_TS: &'static str = "to_ts";
+
 // definition of default database values
 pub const DEFAULT_PROCESS_ID: &'static str = "default";
-pub const MAX_NUM_RESPONSE_ENTRIES: u64 = 10000;
+pub const MAX_NUM_RESPONSE_ENTRIES: u64 = 1000;
 pub const DEFAULT_NUM_RESPONSE_ENTRIES: u64 = 100;
 
+pub const DEFAULT_DOC_TYPE: &'static str = "IDS_MESSAGE";
+
 // split string symbols for vec_to_string and string_to_vec
 pub const SPLIT_QUOTE: &'static str = "'";
 pub const SPLIT_SIGN: &'static str = "~";

clearing-house-app/core-lib/src/lib.rs

@@ -15,6 +15,7 @@ pub mod errors {
             HexError(hex::FromHexError);
             Io(::std::io::Error) #[cfg(unix)];
             Mongodb(mongodb::error::Error);
+            MongodbBson(mongodb::bson::de::Error);
             SetLogger(log::SetLoggerError);
             ParseLogLevel(log::ParseLevelError);
             Reqwest(reqwest::Error);

clearing-house-app/core-lib/src/model/document.rs

@@ -5,10 +5,10 @@ use generic_array::GenericArray;
 use std::collections::HashMap;
 use uuid::Uuid;
 use crate::errors::*;
-use crate::constants::{SPLIT_CT, SPLIT_QUOTE, SPLIT_SIGN};
+use crate::constants::{SPLIT_CT};
 use crate::model::new_uuid;
 use crate::model::crypto::{KeyEntry, KeyMap};
-use chrono::Utc;
+use chrono::Local;
 
 #[derive(Clone, Serialize, Deserialize, Debug)]
 pub struct DocumentPart {
@@ -154,7 +154,7 @@ impl Document{
             id: Document::create_uuid(),
             dt_id,
             pid,
-            ts: Utc::now().timestamp(),
+            ts: Local::now().timestamp(),
             tc,
             parts,
         }
@@ -170,31 +170,6 @@ impl Document{
             parts,
         }
     }
-
-    pub fn string_to_vec(value: String) -> Option<Vec<String>> {
-        let mut vec = vec![];
-        for item in value.split(SPLIT_SIGN) {
-            match item {
-                "" => (),
-                _ => vec.push(String::from(item))
-            }
-        }
-        if vec.len() > 0 {
-            Some(vec)
-        }
-        else {
-            None
-        }
-    }
-
-    pub fn vec_to_string<T: ToString>(v: Vec<T>) -> String {
-        let mut value = String::new();
-        for x in v {
-            value.push_str(&format!("{}{}{}{}", SPLIT_QUOTE, x.to_string(), SPLIT_QUOTE, SPLIT_SIGN));
-        }
-        let _remove_last = value.pop();
-        value
-    }
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize)]

clearing-house-app/core-lib/src/model/mod.rs

@@ -1,6 +1,7 @@
 use std::sync::RwLock;
 use biscuit::Empty;
 use biscuit::jwk::JWKSet;
+use chrono::{Datelike, Duration, Local, NaiveDate, NaiveDateTime, NaiveTime};
 
 pub mod crypto;
 pub mod document;
@@ -34,4 +35,87 @@ impl JwksCache{
             jwks: RwLock::new(None)
         }
     }
+}
+
+pub fn parse_date(date: Option<String>, to_date: bool) -> Option<NaiveDateTime>{
+    let time_format;
+    if to_date{
+        time_format = "23:59:59"
+    }
+    else{
+        time_format = "00:00:00"
+    }
+
+    match date{
+        Some(d) => {
+            debug!("Parsing date: {}", &d);
+            match NaiveDateTime::parse_from_str(format!("{} {}",&d, &time_format).as_str(), "%Y-%m-%d %H:%M:%S"){
+                Ok(date) => {
+                    Some(date)
+                }
+                Err(e) => {
+                    error!("Error occurred: {:#?}", e);
+                    return None
+                }
+            }
+        }
+        None => None
+    }
+}
+
+pub fn sanitize_dates(date_from: Option<NaiveDateTime>, date_to: Option<NaiveDateTime>) -> (NaiveDateTime, NaiveDateTime){
+    let default_to_date = Local::now().naive_local();
+    let d = NaiveDate::from_ymd(default_to_date.year(), default_to_date.month(), default_to_date.day());
+    let t = NaiveTime::from_hms(0, 0, 0);
+    let default_from_date = NaiveDateTime::new(d,t) - Duration::weeks(2);
+
+    println!("date_to: {:#?}", date_to);
+    println!("date_from: {:#?}", date_from);
+
+    println!("Default date_to: {:#?}", default_to_date);
+    println!("Default date_from: {:#?}", default_from_date);
+
+    // validate already checked that date_from > date_to
+    if date_from.is_some() && date_to.is_some(){
+        return (date_from.unwrap(), date_to.unwrap())
+    }
+
+    // if to_date is missing, default to now
+    if date_from.is_some() && date_to.is_none(){
+        return (date_from.unwrap(), default_to_date)
+    }
+
+    // if both dates are none (case to_date is none and from_date is_some should be catched by validation)
+    // return dates for default duration (last 2 weeks)
+    return (default_from_date, default_to_date)
+}
+
+pub fn validate_dates(date_from: Option<NaiveDateTime>, date_to: Option<NaiveDateTime>) -> bool{
+    let date_now = Local::now().naive_local();
+    debug!("... validating dates: now: {:#?} , from: {:#?} , to: {:#?}", &date_now, &date_from, &date_to);
+    // date_from before now
+    if date_from.is_some() && date_from.as_ref().unwrap().clone() > date_now{
+        debug!("oh no, date_from {:#?} is in the future! date_now is {:#?}", &date_from, &date_now);
+        return false;
+    }
+
+    // date_to only if there is also date_from
+    if date_from.is_none() && date_to.is_some() {
+        return false;
+    }
+
+    // date_to before or equals now
+    if date_to.is_some() && date_to.as_ref().unwrap().clone() >= date_now{
+        debug!("oh no, date_to {:#?} is in the future! date_now is {:#?}", &date_to, &date_now);
+        return false;
+    }
+
+    // date_from before date_to
+    if date_from.is_some() && date_to.is_some(){
+        if date_from.unwrap() > date_to.unwrap() {
+            debug!("oh no, date_from {:#?} is before date_to {:#?}", &date_from, &date_to);
+            return false;
+        }
+    }
+    return true;
 }

clearing-house-app/document-api/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "document-api"
-version = "0.8.0"
+version = "0.9.0"
 authors = [
     "Mark Gall <mark.gall@aisec.fraunhofer.de>",
     "Georg Bramm <georg.bramm@aisec.fraunhofer.de>",
@@ -10,16 +10,17 @@ edition = "2018"
 [dependencies]
 biscuit = { git = "https://github.com/lawliet89/biscuit", branch = "master" }
 chrono = { version = "0.4", features = ["serde"] }
+core-lib = {path = "../core-lib" }
 error-chain = "0.12.4"
 fern = "0.5"
+futures = "0.3.24"
 hex = "0.4.3"
 log = "0.4.14"
-mongodb ="2.0.1"
+mongodb ="2.3.0"
 rocket = { version = "0.5.0-rc.1", features = ["json"] }
 rocket_cors = { git = "https://github.com/lawliet89/rocket_cors", branch = "master" }
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
 tokio = "1.8.1"
 tokio-test = "0.4.2"
-core-lib = {path = "../core-lib" }