From 4d124892e1b1c90ce25ac711de5beb9599203c50 Mon Sep 17 00:00:00 2001 From: "Yuanda (Yenda) Li" <51709729+yuandagits@users.noreply.github.com> Date: Tue, 3 Dec 2024 14:27:45 -0500 Subject: [PATCH 1/4] Create config for setting Native op trace directory config. --- .../main/sphinx/presto_cpp/properties-session.rst | 12 +++++++++++- .../NativeWorkerSessionPropertyProvider.java | 7 ++++++- .../presto_cpp/main/SessionProperties.cpp | 10 ++++++++++ .../presto_cpp/main/SessionProperties.h | 6 ++++++ 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst b/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst index 060158c89136f..6c692b4eee130 100644 --- a/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst +++ b/presto-docs/src/main/sphinx/presto_cpp/properties-session.rst @@ -304,4 +304,14 @@ Use ``0`` to disable prefix-sort. * **Default value:** ``130`` Minimum number of rows to use prefix-sort. -The default value has been derived using micro-benchmarking. \ No newline at end of file +The default value has been derived using micro-benchmarking. + +``native_op_trace_directory_create_config`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``varchar`` +* **Default value:** ``""`` + +Native Execution only. Config used to create operator trace directory. This config is provided +to underlying file system and the config is free form. The form should be defined by the +underlying file system. \ No newline at end of file diff --git a/presto-main/src/main/java/com/facebook/presto/sessionpropertyproviders/NativeWorkerSessionPropertyProvider.java b/presto-main/src/main/java/com/facebook/presto/sessionpropertyproviders/NativeWorkerSessionPropertyProvider.java index 15a2404b4d1b0..3337e28c1c0af 100644 --- a/presto-main/src/main/java/com/facebook/presto/sessionpropertyproviders/NativeWorkerSessionPropertyProvider.java +++ b/presto-main/src/main/java/com/facebook/presto/sessionpropertyproviders/NativeWorkerSessionPropertyProvider.java @@ -65,6 +65,7 @@ public class NativeWorkerSessionPropertyProvider public static final String NATIVE_SPILL_PREFIXSORT_ENABLED = "native_spill_prefixsort_enabled"; public static final String NATIVE_PREFIXSORT_NORMALIZED_KEY_MAX_BYTES = "native_prefixsort_normalized_key_max_bytes"; public static final String NATIVE_PREFIXSORT_MIN_ROWS = "native_prefixsort_min_rows"; + public static final String NATIVE_OP_TRACE_DIR_CREATE_CONFIG = "native_op_trace_directory_create_config"; private final List> sessionProperties; @Inject @@ -130,7 +131,7 @@ public NativeWorkerSessionPropertyProvider(FeaturesConfig featuresConfig) longProperty( NATIVE_WRITER_FLUSH_THRESHOLD_BYTES, "Native Execution only. Minimum memory footprint size required to reclaim memory from a file " + - "writer by flushing its buffered data to disk.", + "writer by flushing its buffered data to disk.", 96L << 20, false), booleanProperty( @@ -226,6 +227,10 @@ public NativeWorkerSessionPropertyProvider(FeaturesConfig featuresConfig) "The regexp of traced task id. We only enable trace on a task if its id matches.", "", !nativeExecution), + stringProperty(NATIVE_OP_TRACE_DIR_CREATE_CONFIG, + "Config used to create operator trace directory. This config is provided to underlying file system and the config is free form. The form should be defined by the underlying file system.", + "", + !nativeExecution), longProperty(NATIVE_MAX_OUTPUT_BUFFER_SIZE, "The maximum size in bytes for the task's buffered output. The buffer is shared among all drivers.", 200L << 20, diff --git a/presto-native-execution/presto_cpp/main/SessionProperties.cpp b/presto-native-execution/presto_cpp/main/SessionProperties.cpp index 85e5d9be2ee0c..1c50d2d91d95a 100644 --- a/presto-native-execution/presto_cpp/main/SessionProperties.cpp +++ b/presto-native-execution/presto_cpp/main/SessionProperties.cpp @@ -306,6 +306,16 @@ SessionProperties::SessionProperties() { QueryConfig::kQueryTraceTaskRegExp, c.queryTraceTaskRegExp()); + addSessionProperty( + kOpTraceDirectoryCreateConfig, + "Config used to create operator trace directory. This config is provided to" + " underlying file system and the config is free form. The form should be defined " + "by the underlying file system.", + VARCHAR(), + false, + QueryConfig::kOpTraceDirectoryCreateConfig, + c.opTraceDirectoryCreateConfig()); + addSessionProperty( kMaxOutputBufferSize, "The maximum size in bytes for the task's buffered output. The buffer is" diff --git a/presto-native-execution/presto_cpp/main/SessionProperties.h b/presto-native-execution/presto_cpp/main/SessionProperties.h index 13a8f51e3d7a4..4d6fb3d4c01e2 100644 --- a/presto-native-execution/presto_cpp/main/SessionProperties.h +++ b/presto-native-execution/presto_cpp/main/SessionProperties.h @@ -207,6 +207,12 @@ class SessionProperties { static constexpr const char* kQueryTraceTaskRegExp = "native_query_trace_task_reg_exp"; + /// Config used to create operator trace directory. This config is provided to + /// underlying file system and the config is free form. The form should be + /// defined by the underlying file system. + static constexpr const char* kOpTraceDirectoryCreateConfig = + "native_op_trace_directory_create_config"; + /// The maximum size in bytes for the task's buffered output. The buffer is /// shared among all drivers. static constexpr const char* kMaxOutputBufferSize = From 6a12d644a30f63fde0eb225eefa6d3eb2737b952 Mon Sep 17 00:00:00 2001 From: Sumi Mathew Date: Wed, 4 Dec 2024 16:05:06 +0530 Subject: [PATCH 2/4] Revert of commit 4f159c030dc97b7a5d40bbeb450db3a99c2f798b --- presto-pinot-toolkit/pom.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/presto-pinot-toolkit/pom.xml b/presto-pinot-toolkit/pom.xml index 6bb0de87cf9a5..03a9840f5538e 100644 --- a/presto-pinot-toolkit/pom.xml +++ b/presto-pinot-toolkit/pom.xml @@ -314,10 +314,6 @@ com.fasterxml.jackson.dataformat jackson-dataformat-yaml - - org.webjars - swagger-ui - From 0610c4b097ff9361692d6ce94b76229c11fff483 Mon Sep 17 00:00:00 2001 From: Sumi Mathew Date: Wed, 4 Dec 2024 16:09:29 +0530 Subject: [PATCH 3/4] Exclude org.webjars:swagger-ui:3.23.11 to fix CVEs --- presto-pinot-toolkit/pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/presto-pinot-toolkit/pom.xml b/presto-pinot-toolkit/pom.xml index 03a9840f5538e..332f3b7f6acd8 100644 --- a/presto-pinot-toolkit/pom.xml +++ b/presto-pinot-toolkit/pom.xml @@ -262,6 +262,10 @@ io.netty netty-transport-native-epoll + + org.webjars + swagger-ui + From 4c44dcfb8fb502ce2a7e07229e640514c7215820 Mon Sep 17 00:00:00 2001 From: Sayari Mukherjee Date: Mon, 2 Dec 2024 17:48:23 +0530 Subject: [PATCH 4/4] Exclude org.hibernate.validator:hibernate-validator:6.2.0.Final to fix CVEs As per maven repository version 6.2.0.Final is vulnerable(https://mvnrepository.com/artifact/org.hibernate.validator/hibernate-validator/6.2.0.Final) --- presto-benchto-benchmarks/pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/presto-benchto-benchmarks/pom.xml b/presto-benchto-benchmarks/pom.xml index 07afc74cce664..ff0add3a43dc6 100644 --- a/presto-benchto-benchmarks/pom.xml +++ b/presto-benchto-benchmarks/pom.xml @@ -47,6 +47,10 @@ org.springframework spring-beans + + org.hibernate + hibernate-validator +