1. 确认Glue作业是否使用正确的数据源和去重逻辑。
2. 修改Glue作业代码，添加去重逻辑。以下是一些示例代码，用于去除Athena表格中的重复记录：

import sys from awsglue.transforms import * from awsglue.utils import getResolvedOptions from awsglue.context import GlueContext from pyspark.context import SparkContext from pyspark.sql import SparkSession

Create Spark session

spark = SparkSession.builder.appName("DuplicateRecordRemover").getOrCreate()

table_name = "my_table_name" output_location = "s3://my-bucket-name/my-path"

Create Glue context

glueContext = GlueContext(SparkContext.getOrCreate())

Create dynamic frame

my_dyframe = glueContext.create_dynamic_frame.from_catalog(database="my_database_name", table_name=table_name)

Convert dynamic frame to data frame

my_df = my_dyframe.toDF()

Remove duplicates

my_df = my_df.drop_duplicates()

Write data frame to output location

my_df.write.mode('append').parquet(output_location)

Convert data frame to dynamic frame

my_dyframe = glueContext.create_dynamic_frame.from_options(frame=my_df, connection_type="s3", connection_options={"path": output_location})

Write dynamic frame to Athena table

glueContext.write_dynamic_frame.from_catalog(frame=my_dyframe, database="my_database_name", table_name=table_name)