-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathsofodata.py
More file actions
120 lines (109 loc) · 4.1 KB
/
Copy pathsofodata.py
File metadata and controls
120 lines (109 loc) · 4.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
import os
import json
import requests
import random
import string
def to_api(client_id, client_secret, dataframe, name, description, debug=False):
api_endpoint = 'https://api.sofodata.com'
tmp_file_location = '/tmp/' + get_random_alphanumeric_string(32) + '.csv'
# Step 1 Save the dataframe to temp csv file
# https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_csv.html
if debug:
print("Saving Dataframe to temp file " + tmp_file_location)
dataframe.to_csv(path_or_buf=tmp_file_location, index=False)
# Step 2 - Calculate the column headers
# https://pbpython.com/pandas_dtypes.html
column_headers = []
for i in range(len(dataframe.columns)):
column = dataframe.columns[i]
if dataframe[column].dtype == 'bool':
column_headers.append({
"name": column,
"type": "BOOLEAN",
'indexed': i == 0
})
elif dataframe[column].dtype == 'int64':
column_headers.append({
"name": column,
"type": "NUMBER",
'indexed': i == 0
})
elif dataframe[column].dtype == 'float64':
column_headers.append({
"name": column,
"type": "DECIMAL",
'indexed': i == 0
})
else:
column_headers.append({
"name": column,
"type": "STRING",
'indexed': i == 0
})
# Step 3 - Create an OAuth access token
if debug:
print("Getting OAuth Access Token")
url = api_endpoint + '/v8/oauth/token'
headers = {'content-type': 'application/json'}
payload = {
"client_id": client_id,
"client_secret": client_secret,
"audience": "https://api.sofodata.com/",
"grant_type": "client_credentials"
}
r = requests.post(url, headers=headers, data=json.dumps(payload))
if debug:
print(r.status_code)
print("Response: " + r.text)
access_token = json.loads(r.text)['access_token']
# Step 4 - Generate a S3 upload signature & policy
if debug:
print("Generate S3 Upload Signature & Policy")
url = api_endpoint + '/v8/signature/policy'
headers = {'content-type': 'application/json', 'authorization': 'Bearer ' + access_token}
r = requests.post(url, headers=headers, data=json.dumps(payload))
if debug:
print(r.status_code)
print("Response: " + r.text)
upload_response = json.loads(r.text)
s3_upload_url = upload_response['s3UploadURL']
s3_bucket_name = upload_response['s3BucketName']
s3_object_key = upload_response['s3ObjectKey']
s3_policy_document = upload_response['policyDocument']
# Step 5 - Upload the file to S3
if debug:
print("Uploading File to S3")
files = {'file': open(tmp_file_location, 'rb')}
r = requests.post(s3_upload_url, files=files, data=s3_policy_document)
if debug:
print(r.status_code)
print("Response: " + r.text)
# Step 6 Delete to temp csv file
if debug:
print("Deleting temp file " + tmp_file_location)
os.remove(tmp_file_location)
# Step 6 - Create a dataset and triggering a deployment
if debug:
print("Creating Dataset to Deploy")
url = api_endpoint + '/v8/dataSets'
headers = {'content-type': 'application/json', 'authorization': 'Bearer ' + access_token}
payload = {
'name': name,
'description': description,
'status': 'PENDING_DEPLOYMENT',
's3BucketName': s3_bucket_name,
's3ObjectKey': s3_object_key,
'fileType': 'CSV',
'fileContainsHeader': True,
'columnDelimiter': 'COMMA_SEPARATED',
'columnHeaders': column_headers
}
r = requests.post(url, headers=headers, data=json.dumps(payload))
if debug:
print(r.status_code)
print("Response: " + r.text)
return json.loads(r.text)
def get_random_alphanumeric_string(length):
letters = string.ascii_lowercase + string.digits
result_str = ''.join(random.choice(letters) for i in range(length))
return result_str