Daily I am loading some files. Each file is loading into 2 tables based on some filters. For example Table1.C1 column values is null python script must go to second Table script. cursor.execute("update Fact StagingTable1 set Is_Active=0 where C1 = ?", C1) At the above line script is failing when C1 is null.
How to write the code? Please help me.
This is the original script I used
import os, sys, time
import glob
import shutil
import os
import pandas as pd
import time
import numpy as np
import pyodbc
todaysdate = time.strftime("%Y%m%d")
folder_name ="C:\\Path"
dest = "C:\\Path\\Archive\\"
file_type = ''
#all_Sales_files = sorted(glob.glob(folder_name + "/*Sales_*" ))
#all_Sales_files_sorted=sorted(os.listdir(folder_name))
file_titles = ['Col1']
os.chdir(folder_name)
all_Sales_files = filter(os.path.isfile, os.listdir(folder_name))
all_Sales_files = [os.path.join(folder_name, filename) for filename in all_Sales_files] # add path to each file
all_Sales_files.sort(key=lambda x: os.path.getmtime(x))
for filename in all_Sales_files:
base = os.path.basename(filename)
basewoe = os.path.splitext(base)[0]
print(base + " File is started processing ...")
## File1 is processing into Table1 base on conditions
df=pd.read_csv(filename, header=None,names=file_titles,low_memory=False)
sql_conn = pyodbc.connect(
"Driver={SQL Server};"
"Server=DB_Server;"
"Database=DB;"
"Trusted_Connection=yes;")
cursor = sql_conn.cursor()
cursor.execute("truncate table StagingTable1")
print(base +': truncate staging table StagingTable1 is successful...')
print(base +': Data is started loading into staging table StagingTable1....')
for index,row in dfd.iterrows():
cursor.execute("INSERT INTO StagingTable1(C1,C2) Values (?,?)",
row['C1'],
row['C2'],
)
sql_conn.commit()
print(base +': Data Loading into staging table StagingTable1 is successful...')
cursor.close()
sql_conn.close()
C1=cursor.execute("select top 1 C1 from StagingTable1")
C1_Result = cursor.fetchall()
C1=[]
for C1 in C1_Result:
print(C1)
cursor.execute("update Fact StagingTable1 set Is_Active=0 where C1 = ?", C1)
sql_insert_query = "insert into FactTable1 select * from StagingTable1"
cursor.execute(sql_insert_query)
sql_conn.commit()
## File1 is processing into Table2 base on conditions
df=pd.read_csv(filename, header=None,names=file_titles,low_memory=False)
sql_conn = pyodbc.connect(
"Driver={SQL Server};"
"Server=DB_Server;"
"Database=DB;"
"Trusted_Connection=yes;")
cursor = sql_conn.cursor()
cursor.execute("truncate table StagingTable1")
print(base +': truncate staging table StagingTable1 is successful...')
print(base +': Data is started loading into staging table StagingTable1....')
for index,row in dfd.iterrows():
cursor.execute("INSERT INTO StagingTable2 (C1,C2) Values (?,?)",
row['C1'],
row['C2'],
)
sql_conn.commit()
print(base +': Data Loading into staging table StagingTable2 is successful...')
cursor.close()
sql_conn.close()
print(base +': Data Loading into table Fact table Fact StagingTable2 is started...')
sql_insert_query = "insert into FactTable2 select * from StagingTable2"
cursor.execute(sql_insert_query)
sql_conn.commit()
shutil.move(filename, dest)
print("Archiving "+ base +" file is successful...")
print("All files of Sales are successfully loaded into tables...")
Aucun commentaire:
Enregistrer un commentaire