Skip to content

Commit

Permalink
lint fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
bradmiro committed Mar 25, 2024
1 parent 97463f2 commit 79753ff
Showing 1 changed file with 9 additions and 6 deletions.
15 changes: 9 additions & 6 deletions src/python/bigtable.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,9 @@

# Create a Spark session and configure the spark-bigtable connector.
spark = SparkSession.builder \
.config('spark.jars',
"gs://spark-bigtable-preview/jars/spark-bigtable-0.0.1-preview5-SNAPSHOT.jar") \
.config("spark.jars",
"gs://spark-bigtable-preview/jars/" +
"spark-bigtable-0.0.1-preview5-SNAPSHOT.jar") \
.getOrCreate()

# Create the catalog schema to convert Bigtable columns to Spark.
Expand Down Expand Up @@ -79,13 +80,15 @@
.options(catalog=catalog) \
.load()


# Create new dfs counting each recommended item per rec position.
# Rename columns to join later.
def groupby_count_rename(df, col):
return df.groupBy(col) \
.count() \
.withColumnRenamed(col, "item") \
.withColumnRenamed("count", col)
return df.groupBy(col) \
.count() \
.withColumnRenamed(col, "item") \
.withColumnRenamed("count", col)


r0 = groupby_count_rename(df, "rec0")
r1 = groupby_count_rename(df, "rec1")
Expand Down

0 comments on commit 79753ff

Please sign in to comment.