Snowflake
def main(): # Initialize Spark session spark = SparkSession.builder \ .appName("SnowflakeToS3") \ .config("spark.hadoop.fs.s3a.connection.timeout", "3600000") \ # 1 hour .config("spark.hadoop.fs.s3a.socket.timeout", "3600000") \ # 1 hour .config("spark.sql.sources.commitProtocolClass", "org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol") \ .config("spark.sql.parquet.output.committer.class", "org.apache.parquet.hadoop.ParquetOutputCommitter") \ .getOrCreate()
def main():
# Initialize Spark session
spark = SparkSession.builder \
.appName("SnowflakeToS3") \
.config("spark.hadoop.fs.s3a.connection.timeout", "3600000") \ # 1 hour
.config("spark.hadoop.fs.s3a.socket.timeout", "3600000") \ # 1 hour
.config("spark.sql.sources.commitProtocolClass", "org.apache.spark.sql.execution.datasources.SQLHadoopMapReduceCommitProtocol") \
.config("spark.sql.parquet.output.committer.class", "org.apache.parquet.hadoop.ParquetOutputCommitter") \
.getOrCreate()
