SNOW-1003123: Revert "Set no-unused-vars rule to error" (#751)

Co-authored-by: Przemyslaw Motacki <przemyslaw.motacki@snowflake.com>

.eslintrc.js

@@ -42,7 +42,7 @@ module.exports = {
     'no-prototype-builtins': ['error'],
     'no-redeclare': ['error'],
     'no-undef': ['error'],
-    'no-unused-vars': ['error'],
+    'no-unused-vars': ['warn'],
     'no-useless-catch': ['error'],
     'no-useless-escape': ['error'],
     'no-var': ['error'],

lib/agent/https_ocsp_agent.js

@@ -2,6 +2,7 @@
  * Copyright (c) 2015-2019 Snowflake Computing Inc. All rights reserved.
  */
 
+const Util = require('../util');
 const HttpsAgent = require('https').Agent;
 const SocketUtil = require('./socket_util');
 
@@ -14,7 +15,7 @@ const SocketUtil = require('./socket_util');
  * @constructor
  */
 function HttpsOcspAgent(options) {
-  const agent = HttpsAgent.apply(this, options);
+  const agent = HttpsAgent.apply(this, arguments);
   agent.createConnection = function (port, host, options) {
     // make sure the 'options' variables references the argument that actually
     // contains the options

lib/authentication/auth_default.js

@@ -22,7 +22,7 @@ function AuthDefault(password) {
     body['data']['PASSWORD'] = password;
   };
 
-  this.authenticate = async function () {
+  this.authenticate = async function (authenticator, serviceName, account, username) {
     return;
   };
 }

lib/authentication/auth_oauth.js

@@ -22,7 +22,7 @@ function AuthOauth(token) {
     body['data']['TOKEN'] = token;
   };
 
-  this.authenticate = async function () {
+  this.authenticate = async function (authenticator, serviceName, account, username) {
     return;
   };
 }

lib/connection/bind_uploader.js

@@ -3,9 +3,11 @@
  */
 const Logger = require('../logger');
 
+const Readable = require('stream').Readable;
 const fs = require('fs');
 
 const Statement = require('./statement');
+const fileCompressionType = require('.././file_transfer_agent/file_compression_type');
 
 const STAGE_NAME = 'SYSTEM$BIND';
 const CREATE_STAGE_STMT = 'CREATE OR REPLACE TEMPORARY STAGE '
@@ -63,7 +65,7 @@ function BindUploader(options, services, connectionConfig, requestId) {
       const putStmt = 'PUT file://' + fileName + '\'' + stageName + '\' overwrite=true auto_compress=false source_compression=gzip';
       const uploadFileOptions = {
         sqlText: putStmt, fileStream: fileData,
-        complete: function (err, stmt) {
+        complete: function (err, stmt, rows) {
           if (err) {
             Logger.getInstance().debug('err ' + err);
             reject(err);

lib/connection/connection.js

@@ -4,6 +4,7 @@
 const { v4: uuidv4 } = require('uuid');
 const Url = require('url');
 const QueryString = require('querystring');
+const GSErrors = require('../constants/gs_errors');
 const QueryStatus = require('../constants/query_status');
 
 const Util = require('../util');

lib/connection/result/column.js

@@ -238,7 +238,7 @@ function createFnIsColumnOfType(columnType, columnComparisonFn, scope) {
  *
  * @returns {Object}
  */
-function convertRawNumber(rawColumnValue) {
+function convertRawNumber(rawColumnValue, column, context) {
   return {
     raw: rawColumnValue,
     processed: Number(rawColumnValue)
@@ -255,7 +255,7 @@ function convertRawNumber(rawColumnValue) {
  * @param context
  * @returns {{processed: bigInt.BigInteger, raw: *}}
  */
-function convertRawBigInt(rawColumnValue) {
+function convertRawBigInt(rawColumnValue, column, context) {
   return {
     raw: rawColumnValue,
     processed: bigInt(rawColumnValue)
@@ -272,7 +272,7 @@ function convertRawBigInt(rawColumnValue) {
  *
  * @returns {Boolean}
  */
-function convertRawBoolean(rawColumnValue) {
+function convertRawBoolean(rawColumnValue, column, context) {
   let ret;
 
   if ((rawColumnValue === '1') || (rawColumnValue === 'TRUE')) {
@@ -485,7 +485,7 @@ function convertRawTimestampHelper(
  *
  * @returns {Object | Array}
  */
-function convertRawVariant(rawColumnValue) {
+function convertRawVariant(rawColumnValue, column, context) {
   // if the input is a non-empty string, convert it to a json object
   if (Util.string.isNotNullOrEmpty(rawColumnValue)) {
     try {

lib/connection/result/result.js

@@ -7,6 +7,7 @@ const Util = require('../../util');
 const Errors = require('../../errors');
 const Chunk = require('./chunk');
 const ResultStream = require('./result_stream');
+const ChunkCache = require('./chunk_cache');
 const Column = require('./column');
 const StatementType = require('./statement_type');
 const ColumnNamesCreator = require('./unique_column_name_creator');
@@ -154,6 +155,41 @@ Result.prototype.refresh = function (response) {
   }
 };
 
+/**
+ * TODO
+ *
+ * @param chunks
+ * @param capacity
+ *
+ * @returns {ChunkCache}
+ */
+// eslint-disable-next-line no-unused-vars
+function createChunkCache(chunks, capacity) {
+  let index;
+  let length;
+
+  // create a chunk cache
+  const chunkCache = new ChunkCache(capacity);
+
+  // every time a chunk is loaded, add it to the cache
+  // TODO: should the caching be based on most recently 'used' or most recently
+  // 'loaded'?
+  const onLoadComplete = function (err, chunk) {
+    if (!err) {
+      chunkCache.put(chunk);
+    }
+  };
+
+  // subscribe to the 'loadcomplete' event on all the chunks
+  for (index = 0, length = chunks.length; index < length; index++) {
+    chunks[index].on('loadcomplete', onLoadComplete);
+  }
+
+  // TODO: do we need to unsubscribe from the loadcomplete event at some point?
+
+  return chunkCache;
+}
+
 /**
  * Creates a session state object from the values of the current role, current
  * warehouse, etc., returned in the result response.

lib/connection/result/result_stream.js

@@ -80,7 +80,7 @@ function ResultStream(options) {
       chunks[currChunk].on('loadcomplete', onLoadComplete);
 
       // Fire off requests to load all the chunks in the buffer that aren't already loading
-      let chunk, index;
+      let chunk, index, length;
       for (index = currChunk; index < chunks.length && index <= (currChunk + prefetchSize); index++) {
         chunk = chunks[index];
         if (!chunk.isLoading()) {

lib/connection/statement.js

@@ -58,7 +58,7 @@ exports.createContext = function (
 function createStatement(
   statementOptions, context, services, connectionConfig) {
   // call super
-  BaseStatement.apply(this, [statementOptions, context, services, connectionConfig]);
+  BaseStatement.apply(this, arguments);
 }
 
 /**
@@ -618,7 +618,7 @@ function BaseStatement(
    *
    * @param {Object} body
    */
-  context.onStatementRequestSucc = function () {
+  context.onStatementRequestSucc = function (body) {
   };
 }
 
@@ -686,7 +686,7 @@ function RowStatementPreExec(
   connectionConfig) {
   Logger.getInstance().debug('RowStatementPreExec');
   // call super
-  BaseStatement.apply(this, [statementOptions, context, services, connectionConfig]);
+  BaseStatement.apply(this, arguments);
 
   // add the result request headers to the context
   context.resultRequestHeaders = buildResultRequestHeadersRow();
@@ -788,7 +788,7 @@ function createOnStatementRequestSuccRow(statement, context) {
 function FileStatementPreExec(
   statementOptions, context, services, connectionConfig) {
   // call super
-  BaseStatement.apply(this, [statementOptions, context, services, connectionConfig]);
+  BaseStatement.apply(this, arguments);
 
   // add the result request headers to the context
   context.resultRequestHeaders = buildResultRequestHeadersFile();
@@ -869,7 +869,7 @@ function StageBindingStatementPreExec(
    *
    * @param {Object} body
    */
-  context.onStatementRequestSucc = function () {
+  context.onStatementRequestSucc = function (body) {
     //do nothing
   };
 
@@ -933,7 +933,7 @@ Util.inherits(StageBindingStatementPreExec, BaseStatement);
 function StatementPostExec(
   statementOptions, context, services, connectionConfig) {
   // call super
-  BaseStatement.apply(this, [statementOptions, context, services, connectionConfig]);
+  BaseStatement.apply(this, arguments);
 
   // add the result request headers to the context
   context.resultRequestHeaders = buildResultRequestHeadersRow();
@@ -1314,7 +1314,7 @@ this.sendRequest = function (statementContext, onResultAvailable) {
   // clone the options
   options = Util.apply({}, options);
 
-  return new Promise((resolve) => {
+  return new Promise((resolve, reject) => {
     resolve(sf.postAsync(options));
   });
 };
@@ -1585,13 +1585,13 @@ function countBinding(binds) {
 }
 
 function hasNextResult(statement, context) {
-  return function () {
+  return function (options) {
     return (context.multiResultIds != null && context.multiCurId + 1 < context.multiResultIds.length);
   };
 }
 
 function createNextReuslt(statement, context) {
-  return function () {
+  return function (options) {
     if (hasNextResult(statement, context)) {
       context.multiCurId++;
       context.queryId = context.multiResultIds[context.multiCurId];

lib/core.js

@@ -267,7 +267,7 @@ function Core(options) {
     */
     this.destroy = function (connection) {
       return new Promise((resolve) => {
-        connection.destroy(function (err) {
+        connection.destroy(function (err, conn) {
           if (err) {
             Logger.getInstance().error('Unable to disconnect: ' + err.message);
           }

lib/errors.js

@@ -575,7 +575,7 @@ function createError(name, options) {
 
   // if the error is not synchronous, add an externalize() method
   if (!options.synchronous) {
-    error.externalize = function () {
+    error.externalize = function (errorCode, errorMessageArgs, sqlState) {
       const propNames =
         [
           'name',

lib/file_transfer_agent/azure_util.js

@@ -158,10 +158,11 @@ function AzureUtil(azure, filestream) {
   * @param {String} fileStream
   * @param {Object} meta
   * @param {Object} encryptionMetadata
+  * @param {Number} maxConcurrency
   *
   * @returns {null}
   */
-  this.uploadFileStream = async function (fileStream, meta, encryptionMetadata) {
+  this.uploadFileStream = async function (fileStream, meta, encryptionMetadata, maxConcurrency) {
     const azureMetadata = {
       'sfcdigest': meta['SHA256_DIGEST']
     };
@@ -223,12 +224,14 @@ function AzureUtil(azure, filestream) {
   /**
   * Download the file blob then write the file.
   *
+  * @param {String} dataFile
   * @param {Object} meta
-  * @param {Object} fullDstPath
+  * @param {Object} encryptionMetadata
+  * @param {Number} maxConcurrency
   *
   * @returns {null}
   */
-  this.nativeDownloadFile = async function (meta, fullDstPath) {
+  this.nativeDownloadFile = async function (meta, fullDstPath, maxConcurrency) {
     const stageInfo = meta['stageInfo'];
     const client = this.createClient(stageInfo);
     const azureLocation = this.extractContainerNameAndPath(stageInfo['location']);

lib/file_transfer_agent/encrypt_util.js

@@ -129,7 +129,8 @@ function EncryptUtil(encrypt, filestream, temp) {
   *
   * @returns {Object}
   */
-  this.encryptFileStream = async function (encryptionMaterial, fileStream) {
+  this.encryptFileStream = async function (encryptionMaterial, fileStream,
+    tmpDir = null, chunkSize = blockSize * 4 * 1024) {
     // Get decoded key from base64 encoded value
     const decodedKey = Buffer.from(encryptionMaterial[QUERY_STAGE_MASTER_KEY], BASE64);
     const keySize = decodedKey.length;
@@ -197,7 +198,7 @@ function EncryptUtil(encrypt, filestream, temp) {
     const tempOutputFileName = tmpobj.name;
     const tempFd = tmpobj.fd;
 
-    await new Promise(function (resolve) {
+    await new Promise(function (resolve, reject) {
       const infile = fs.createReadStream(inFileName, { highWaterMark: chunkSize });
       const outfile = fs.createWriteStream(tempOutputFileName);
 
@@ -290,7 +291,7 @@ function EncryptUtil(encrypt, filestream, temp) {
     // Create decipher with file key, iv bytes, and AES CBC
     decipher = crypto.createDecipheriv(AES_CBC, fileKey, ivBytes);
 
-    await new Promise(function (resolve) {
+    await new Promise(function (resolve, reject) {
       const infile = fs.createReadStream(inFileName, { highWaterMark: chunkSize });
       const outfile = fs.createWriteStream(tempOutputFileName);
 

lib/file_transfer_agent/file_util.js

@@ -54,7 +54,7 @@ function FileUtil() {
     const baseName = path.basename(fileName);
     const gzipFileName = path.join(tmpDir, baseName + '_c.gz');
 
-    await new Promise(function (resolve) {
+    await new Promise(function (resolve, reject) {
       // Create gzip object
       const gzip = zlib.createGzip();
       // Create stream object for reader and writer
@@ -110,7 +110,7 @@ function FileUtil() {
     const bufferSize = fileInfo.size;
 
     let buffer = [];
-    await new Promise(function (resolve) {
+    await new Promise(function (resolve, reject) {
       // Create reader stream and set maximum chunk size
       const infile = fs.createReadStream(fileName, { highWaterMark: chunkSize });
       infile.on('data', function (chunk) {

lib/file_transfer_agent/gcs_util.js

@@ -15,8 +15,10 @@ const GCS_METADATA_ENCRYPTIONDATAPROP = GCS_METADATA_PREFIX + ENCRYPTIONDATAPROP
 const GCS_FILE_HEADER_DIGEST = 'gcs-file-header-digest';
 const GCS_FILE_HEADER_CONTENT_LENGTH = 'gcs-file-header-content-length';
 const GCS_FILE_HEADER_ENCRYPTION_METADATA = 'gcs-file-header-encryption-metadata';
+const CONTENT_CHUNK_SIZE = 10 * 1024;
 
 const HTTP_HEADER_CONTENT_ENCODING = 'Content-Encoding';
+const HTTP_HEADER_ACCEPT_ENCODING = 'Accept-Encoding';
 const resultStatus = require('./file_util').resultStatus;
 
 const { Storage } = require('@google-cloud/storage');
@@ -221,10 +223,11 @@ function GCSUtil(httpclient, filestream) {
     * @param {String} fileStream
     * @param {Object} meta
     * @param {Object} encryptionMetadata
+    * @param {Number} maxConcurrency
     *
     * @returns {null}
     */
-  this.uploadFileStream = async function (fileStream, meta, encryptionMetadata) {
+  this.uploadFileStream = async function (fileStream, meta, encryptionMetadata, maxConcurrency) {
     let uploadUrl = meta['presignedUrl'];
     let accessToken = null;
 
@@ -326,12 +329,14 @@ function GCSUtil(httpclient, filestream) {
   /**
    * Download the file.
    *
+   * @param {String} dataFile
    * @param {Object} meta
-   * @param {Object} fullDstPath
+   * @param {Object} encryptionMetadata
+   * @param {Number} maxConcurrency
    *
    * @returns {null}
    */
-  this.nativeDownloadFile = async function (meta, fullDstPath) {
+  this.nativeDownloadFile = async function (meta, fullDstPath, maxConcurrency) {
     let downloadUrl = meta['presignedUrl'];
     let accessToken = null;
     let gcsHeaders = {};

lib/file_transfer_agent/local_util.js

@@ -14,7 +14,7 @@ const resultStatus = require('./file_util').resultStatus;
  * @constructor
  */
 function LocalUtil() {
-  this.createClient = function () {
+  this.createClient = function (stageInfo, useAccelerateEndpoint) {
     return null;
   };