diff --git a/bindings/nodejs/Cargo.toml b/bindings/nodejs/Cargo.toml index c6ea08df0967..2f234028f1ee 100644 --- a/bindings/nodejs/Cargo.toml +++ b/bindings/nodejs/Cargo.toml @@ -45,6 +45,10 @@ default = [ "services-webhdfs", ] + +# NOTE: this is the feature we used to build pypi wheels. +# When enable or disable some features, +# also need to update dev/src/generate/binding_nodejs.rs `enabled_service` to match it. services-all = [ "default", "services-aliyun-drive", diff --git a/bindings/nodejs/index.d.ts b/bindings/nodejs/index.d.ts index 8c22b10e14f8..ad6235e40bb2 100644 --- a/bindings/nodejs/index.d.ts +++ b/bindings/nodejs/index.d.ts @@ -41,3 +41,4 @@ declare module './generated' { } export * from './generated' +export { create_operator } from './types.generated' diff --git a/bindings/nodejs/index.js b/bindings/nodejs/index.js index 22ef628e09b4..2581fefc4ccc 100644 --- a/bindings/nodejs/index.js +++ b/bindings/nodejs/index.js @@ -137,6 +137,10 @@ Writer.prototype.createWriteStream = function (options) { return new WriteStream(this, options) } +module.exports.create_operator = function (scheme, options) { + return new Operator(scheme, options) +} + module.exports.Operator = Operator module.exports.layers = { RetryLayer, diff --git a/bindings/nodejs/package.json b/bindings/nodejs/package.json index aa3f22a6d48f..90b921af4afa 100644 --- a/bindings/nodejs/package.json +++ b/bindings/nodejs/package.json @@ -39,6 +39,7 @@ "webhdfs" ], "files": [ + "types.generated.d.ts", "index.d.ts", "index.js", "generated.d.ts", diff --git a/bindings/nodejs/types.generated.d.ts b/bindings/nodejs/types.generated.d.ts new file mode 100644 index 000000000000..d1c539a08307 --- /dev/null +++ b/bindings/nodejs/types.generated.d.ts @@ -0,0 +1,584 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Operator } from './generated' + +export interface Aliyun_drive_Config { + root?: string + access_token?: string + client_id?: string + client_secret?: string + refresh_token?: string + drive_type: string +} + +export interface Alluxio_Config { + root?: string + endpoint?: string +} + +export interface Atomicserver_Config { + root?: string + endpoint?: string + private_key?: string + public_key?: string + parent_resource_id?: string +} + +export interface Azblob_Config { + root?: string + container: string + endpoint?: string + account_name?: string + account_key?: string + encryption_key?: string + encryption_key_sha256?: string + encryption_algorithm?: string + sas_token?: string + batch_max_operations?: string +} + +export interface Azdls_Config { + root?: string + filesystem: string + endpoint?: string + account_name?: string + account_key?: string +} + +export interface Azfile_Config { + root?: string + endpoint?: string + share_name: string + account_name?: string + account_key?: string + sas_token?: string +} + +export interface B2_Config { + root?: string + application_key_id?: string + application_key?: string + bucket: string + bucket_id: string +} + +export interface Cacache_Config { + datadir?: string +} + +export interface Chainsafe_Config { + root?: string + api_key?: string + bucket_id: string +} + +export interface Cloudflare_kv_Config { + token?: string + account_id?: string + namespace_id?: string + root?: string +} + +export interface Compfs_Config { + root?: string +} + +export interface Cos_Config { + root?: string + endpoint?: string + secret_id?: string + secret_key?: string + bucket?: string + disable_config_load?: string +} + +export interface D1_Config { + token?: string + account_id?: string + database_id?: string + root?: string + table?: string + key_field?: string + value_field?: string +} + +export interface Dashmap_Config { + root?: string +} + +export interface Dbfs_Config { + root?: string + endpoint?: string + token?: string +} + +export interface Dropbox_Config { + root?: string + access_token?: string + refresh_token?: string + client_id?: string + client_secret?: string +} + +export interface Fs_Config { + root?: string + atomic_write_dir?: string +} + +export interface Gcs_Config { + root?: string + bucket: string + endpoint?: string + scope?: string + service_account?: string + credential?: string + credential_path?: string + predefined_acl?: string + default_storage_class?: string + allow_anonymous?: string + disable_vm_metadata?: string + disable_config_load?: string + token?: string +} + +export interface Gdrive_Config { + root?: string + access_token?: string + refresh_token?: string + client_id?: string + client_secret?: string +} + +export interface Ghac_Config { + root?: string + version?: string + endpoint?: string + runtime_token?: string +} + +export interface Github_Config { + root?: string + token?: string + owner: string + repo: string +} + +export interface Gridfs_Config { + connection_string?: string + database?: string + bucket?: string + chunk_size?: string + root?: string +} + +export interface Hdfs_native_Config { + root?: string + url?: string + enable_append?: string +} + +export interface Http_Config { + endpoint?: string + username?: string + password?: string + token?: string + root?: string +} + +export interface Huggingface_Config { + repo_type?: string + repo_id?: string + revision?: string + root?: string + token?: string +} + +export interface Icloud_Config { + root?: string + apple_id?: string + password?: string + trust_token?: string + ds_web_auth_token?: string + is_china_mainland?: string +} + +export interface Ipfs_Config { + endpoint?: string + root?: string +} + +export interface Ipmfs_Config { + root?: string + endpoint?: string +} + +export interface Koofr_Config { + root?: string + endpoint: string + email: string + password?: string +} + +export interface Lakefs_Config { + endpoint?: string + username?: string + password?: string + root?: string + repository?: string + branch?: string +} + +export interface Libsql_Config { + connection_string?: string + auth_token?: string + table?: string + key_field?: string + value_field?: string + root?: string +} + +export interface Memcached_Config { + endpoint?: string + root?: string + username?: string + password?: string + default_ttl?: string +} + +export interface Memory_Config { + root?: string +} + +export interface Mini_moka_Config { + max_capacity?: string + time_to_live?: string + time_to_idle?: string + root?: string +} + +export interface Moka_Config { + name?: string + max_capacity?: string + time_to_live?: string + time_to_idle?: string + num_segments?: string + root?: string +} + +export interface Mongodb_Config { + connection_string?: string + database?: string + collection?: string + root?: string + key_field?: string + value_field?: string +} + +export interface Monoiofs_Config { + root?: string +} + +export interface Mysql_Config { + connection_string?: string + table?: string + key_field?: string + value_field?: string + root?: string +} + +export interface Nebula_graph_Config { + host?: string + port?: string + username?: string + password?: string + space?: string + tag?: string + key_field?: string + value_field?: string + root?: string +} + +export interface Obs_Config { + root?: string + endpoint?: string + access_key_id?: string + secret_access_key?: string + bucket?: string +} + +export interface Onedrive_Config { + access_token?: string + root?: string +} + +export interface Oss_Config { + root?: string + endpoint?: string + presign_endpoint?: string + bucket: string + server_side_encryption?: string + server_side_encryption_key_id?: string + allow_anonymous?: string + access_key_id?: string + access_key_secret?: string + + /** + * @deprecated: Please use `delete_max_size` instead of `batch_max_operations` + */ + batch_max_operations?: string + delete_max_size?: string + role_arn?: string + role_session_name?: string + oidc_provider_arn?: string + oidc_token_file?: string + sts_endpoint?: string +} + +export interface Pcloud_Config { + root?: string + endpoint: string + username?: string + password?: string +} + +export interface Persy_Config { + datafile?: string + segment?: string + index?: string +} + +export interface Postgresql_Config { + root?: string + connection_string?: string + table?: string + key_field?: string + value_field?: string +} + +export interface Redb_Config { + datadir?: string + root?: string + table?: string +} + +export interface Redis_Config { + endpoint?: string + cluster_endpoints?: string + username?: string + password?: string + root?: string + db: string + default_ttl?: string +} + +export interface S3_Config { + root?: string + bucket: string + enable_versioning?: string + endpoint?: string + region?: string + access_key_id?: string + secret_access_key?: string + session_token?: string + role_arn?: string + external_id?: string + role_session_name?: string + disable_config_load?: string + disable_ec2_metadata?: string + allow_anonymous?: string + server_side_encryption?: string + server_side_encryption_aws_kms_key_id?: string + server_side_encryption_customer_algorithm?: string + server_side_encryption_customer_key?: string + server_side_encryption_customer_key_md5?: string + default_storage_class?: string + enable_virtual_host_style?: string + + /** + * @deprecated: Please use `delete_max_size` instead of `batch_max_operations` + */ + batch_max_operations?: string + delete_max_size?: string + disable_stat_with_override?: string + checksum_algorithm?: string + disable_write_with_if_match?: string +} + +export interface Seafile_Config { + root?: string + endpoint?: string + username?: string + password?: string + repo_name: string +} + +export interface Sftp_Config { + endpoint?: string + root?: string + user?: string + key?: string + known_hosts_strategy?: string + enable_copy?: string +} + +export interface Sled_Config { + datadir?: string + root?: string + tree?: string +} + +export interface Sqlite_Config { + connection_string?: string + table?: string + key_field?: string + value_field?: string + root?: string +} + +export interface Supabase_Config { + root?: string + bucket: string + endpoint?: string + key?: string +} + +export interface Surrealdb_Config { + connection_string?: string + username?: string + password?: string + namespace?: string + database?: string + table?: string + key_field?: string + value_field?: string + root?: string +} + +export interface Swift_Config { + endpoint?: string + container?: string + root?: string + token?: string +} + +export interface Upyun_Config { + root?: string + bucket: string + operator?: string + password?: string +} + +export interface Vercel_artifacts_Config { + access_token?: string +} + +export interface Vercel_blob_Config { + root?: string + token?: string +} + +export interface Webdav_Config { + endpoint?: string + username?: string + password?: string + token?: string + root?: string + disable_copy?: string +} + +export interface Webhdfs_Config { + root?: string + endpoint?: string + delegation?: string + disable_list_batch?: string + atomic_write_dir?: string +} + +export interface Yandex_disk_Config { + root?: string + access_token: string +} + +export function create_operator(scheme: 'aliyun_drive', options?: Aliyun_drive_Config | undefined | null): Operator +export function create_operator(scheme: 'alluxio', options?: Alluxio_Config | undefined | null): Operator +export function create_operator(scheme: 'atomicserver', options?: Atomicserver_Config | undefined | null): Operator +export function create_operator(scheme: 'azblob', options?: Azblob_Config | undefined | null): Operator +export function create_operator(scheme: 'azdls', options?: Azdls_Config | undefined | null): Operator +export function create_operator(scheme: 'azfile', options?: Azfile_Config | undefined | null): Operator +export function create_operator(scheme: 'b2', options?: B2_Config | undefined | null): Operator +export function create_operator(scheme: 'cacache', options?: Cacache_Config | undefined | null): Operator +export function create_operator(scheme: 'chainsafe', options?: Chainsafe_Config | undefined | null): Operator +export function create_operator(scheme: 'cloudflare_kv', options?: Cloudflare_kv_Config | undefined | null): Operator +export function create_operator(scheme: 'compfs', options?: Compfs_Config | undefined | null): Operator +export function create_operator(scheme: 'cos', options?: Cos_Config | undefined | null): Operator +export function create_operator(scheme: 'd1', options?: D1_Config | undefined | null): Operator +export function create_operator(scheme: 'dashmap', options?: Dashmap_Config | undefined | null): Operator +export function create_operator(scheme: 'dbfs', options?: Dbfs_Config | undefined | null): Operator +export function create_operator(scheme: 'dropbox', options?: Dropbox_Config | undefined | null): Operator +export function create_operator(scheme: 'fs', options?: Fs_Config | undefined | null): Operator +export function create_operator(scheme: 'gcs', options?: Gcs_Config | undefined | null): Operator +export function create_operator(scheme: 'gdrive', options?: Gdrive_Config | undefined | null): Operator +export function create_operator(scheme: 'ghac', options?: Ghac_Config | undefined | null): Operator +export function create_operator(scheme: 'github', options?: Github_Config | undefined | null): Operator +export function create_operator(scheme: 'gridfs', options?: Gridfs_Config | undefined | null): Operator +export function create_operator(scheme: 'hdfs_native', options?: Hdfs_native_Config | undefined | null): Operator +export function create_operator(scheme: 'http', options?: Http_Config | undefined | null): Operator +export function create_operator(scheme: 'huggingface', options?: Huggingface_Config | undefined | null): Operator +export function create_operator(scheme: 'icloud', options?: Icloud_Config | undefined | null): Operator +export function create_operator(scheme: 'ipfs', options?: Ipfs_Config | undefined | null): Operator +export function create_operator(scheme: 'ipmfs', options?: Ipmfs_Config | undefined | null): Operator +export function create_operator(scheme: 'koofr', options?: Koofr_Config | undefined | null): Operator +export function create_operator(scheme: 'lakefs', options?: Lakefs_Config | undefined | null): Operator +export function create_operator(scheme: 'libsql', options?: Libsql_Config | undefined | null): Operator +export function create_operator(scheme: 'memcached', options?: Memcached_Config | undefined | null): Operator +export function create_operator(scheme: 'memory', options?: Memory_Config | undefined | null): Operator +export function create_operator(scheme: 'mini_moka', options?: Mini_moka_Config | undefined | null): Operator +export function create_operator(scheme: 'moka', options?: Moka_Config | undefined | null): Operator +export function create_operator(scheme: 'mongodb', options?: Mongodb_Config | undefined | null): Operator +export function create_operator(scheme: 'monoiofs', options?: Monoiofs_Config | undefined | null): Operator +export function create_operator(scheme: 'mysql', options?: Mysql_Config | undefined | null): Operator +export function create_operator(scheme: 'nebula_graph', options?: Nebula_graph_Config | undefined | null): Operator +export function create_operator(scheme: 'obs', options?: Obs_Config | undefined | null): Operator +export function create_operator(scheme: 'onedrive', options?: Onedrive_Config | undefined | null): Operator +export function create_operator(scheme: 'oss', options?: Oss_Config | undefined | null): Operator +export function create_operator(scheme: 'pcloud', options?: Pcloud_Config | undefined | null): Operator +export function create_operator(scheme: 'persy', options?: Persy_Config | undefined | null): Operator +export function create_operator(scheme: 'postgresql', options?: Postgresql_Config | undefined | null): Operator +export function create_operator(scheme: 'redb', options?: Redb_Config | undefined | null): Operator +export function create_operator(scheme: 'redis', options?: Redis_Config | undefined | null): Operator +export function create_operator(scheme: 's3', options?: S3_Config | undefined | null): Operator +export function create_operator(scheme: 'seafile', options?: Seafile_Config | undefined | null): Operator +export function create_operator(scheme: 'sftp', options?: Sftp_Config | undefined | null): Operator +export function create_operator(scheme: 'sled', options?: Sled_Config | undefined | null): Operator +export function create_operator(scheme: 'sqlite', options?: Sqlite_Config | undefined | null): Operator +export function create_operator(scheme: 'supabase', options?: Supabase_Config | undefined | null): Operator +export function create_operator(scheme: 'surrealdb', options?: Surrealdb_Config | undefined | null): Operator +export function create_operator(scheme: 'swift', options?: Swift_Config | undefined | null): Operator +export function create_operator(scheme: 'upyun', options?: Upyun_Config | undefined | null): Operator +export function create_operator( + scheme: 'vercel_artifacts', + options?: Vercel_artifacts_Config | undefined | null, +): Operator +export function create_operator(scheme: 'vercel_blob', options?: Vercel_blob_Config | undefined | null): Operator +export function create_operator(scheme: 'webdav', options?: Webdav_Config | undefined | null): Operator +export function create_operator(scheme: 'webhdfs', options?: Webhdfs_Config | undefined | null): Operator +export function create_operator(scheme: 'yandex_disk', options?: Yandex_disk_Config | undefined | null): Operator + +export function create_operator(scheme: string, options?: Record | undefined | null): Operator diff --git a/dev/src/generate/binding_nodejs.rs b/dev/src/generate/binding_nodejs.rs new file mode 100644 index 000000000000..d370ea384928 --- /dev/null +++ b/dev/src/generate/binding_nodejs.rs @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use anyhow::Result; +use rinja::Template; +use std::fs; +use std::path::PathBuf; + +use super::parser::Service; +use crate::generate::parser::Services; + +// Using the template in this path, relative +// to the `templates` dir in the crate root +#[derive(Template)] +#[template(path = "types.ts.jinja2", ext = "txt", escape = "none")] +struct PythonTemplate { + services: Vec<(String, Service)>, +} + +/// TODO: add a common utils to parse enabled features from cargo.toml +fn enabled_service(srv: &str) -> bool { + match srv { + // not enabled in bindings/python/Cargo.toml + "etcd" | "foundationdb" | "ftp" | "hdfs" | "rocksdb" | "tikv" => false, + _ => true, + } +} + +pub fn generate(project_root: PathBuf, services: &Services) -> Result<()> { + let v = Vec::from_iter( + services + .clone() + .into_iter() + .filter(|x| enabled_service(x.0.as_str())), + ); + + let tmpl = PythonTemplate { services: v }; + + let s = tmpl.render().expect("should render template"); + + let output_file: String = project_root + .join("bindings/nodejs/types.generated.d.ts") + .to_str() + .expect("should build output file path") + .into(); + + fs::write(output_file, s).expect("failed to write result to file"); + + Ok(()) +} diff --git a/dev/src/generate/mod.rs b/dev/src/generate/mod.rs index 71ec44429998..1726cd1e2c95 100644 --- a/dev/src/generate/mod.rs +++ b/dev/src/generate/mod.rs @@ -17,6 +17,7 @@ mod parser; +mod binding_nodejs; mod binding_python; use anyhow::Result; @@ -30,6 +31,7 @@ pub fn run(language: &str) -> Result<()> { match language { "python" | "py" => binding_python::generate(project_root, &services), + "nodejs" | "js" => binding_nodejs::generate(project_root, &services), _ => Err(anyhow::anyhow!("Unsupported language: {}", language)), } } diff --git a/dev/templates/types.ts.jinja2 b/dev/templates/types.ts.jinja2 new file mode 100644 index 000000000000..a91f32c2137a --- /dev/null +++ b/dev/templates/types.ts.jinja2 @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import { Operator } from "./generated" + +{% for srv in services %} +export interface {{ srv.0|capitalize }}_Config {{'{'}} + {% for field in srv.1.config.clone().into_iter() -%} + {% if field.deprecated.is_some() %} + /** + * @deprecated: {{ field.deprecated.unwrap().note }} + */ + {% endif -%} + {{field.name}} {% if field.optional %}? {% endif %}: string; + {% endfor -%} +{{'}'}} +{% endfor %} + + +{% for srv in services -%} +export function create_operator(scheme: "{{srv.0}}", options?: {{ srv.0|capitalize }}_Config | undefined | null) : Operator; +{% endfor %} + +export function create_operator(scheme: string, options?: Record | undefined | null): Operator; diff --git a/justfile b/justfile index e1f50c6bfcb9..a64b7e8a1fea 100644 --- a/justfile +++ b/justfile @@ -17,6 +17,6 @@ # Generate code for language # -# Available languages: python +# Available languages: python, nodejs generate language: cargo run --quiet --manifest-path=dev/Cargo.toml -- generate -l {{language}}