✨ add snowflake caching ;snowflake:
This commit is contained in:
parent
c0a4e6eee3
commit
76e69d0899
1 changed files with 43 additions and 0 deletions
|
|
@ -1,3 +1,46 @@
|
||||||
|
## database file-caching
|
||||||
|
|
||||||
|
```
|
||||||
|
from pathlib import Path
|
||||||
|
from snowflake.connector import connect
|
||||||
|
import pandas as pd
|
||||||
|
import hashlib
|
||||||
|
|
||||||
|
def get_df(query, env, use_cache=True):
|
||||||
|
print(f'{query[:100] = }')
|
||||||
|
if env == "prod":
|
||||||
|
role = "COSUTMER"
|
||||||
|
warehouse = "CLOWN_COSTUMES"
|
||||||
|
elif env == "sand":
|
||||||
|
role = "CUSTOMER"
|
||||||
|
warehouse = "CUSTOMER_INFORMATION"
|
||||||
|
else:
|
||||||
|
raise ValueError(f"env must be prod or sand - recieved: {env}")
|
||||||
|
|
||||||
|
hash_query_4_digits = hashlib.shake_128(query.encode()).hexdigest(4)
|
||||||
|
cache_pkl_path = Path(f"/tmp/cache-{env}-{hash_query_4_digits}.pkl")
|
||||||
|
print(cache_pkl_path.exists(), cache_pkl_path)
|
||||||
|
if cache_pkl_path.exists() and use_cache:
|
||||||
|
print('loading cached data')
|
||||||
|
df = pd.read_pickle(cache_pkl_path)
|
||||||
|
return df
|
||||||
|
|
||||||
|
print("fetching & caching data")
|
||||||
|
with connect(
|
||||||
|
authenticator="externalbrowser",
|
||||||
|
user="dnth@lundbeck.com",
|
||||||
|
account=f"lundbeck-hlu{env}",
|
||||||
|
role=role,
|
||||||
|
warehouse=warehouse,
|
||||||
|
) as con:
|
||||||
|
cur = con.cursor()
|
||||||
|
cur.execute(query)
|
||||||
|
df = cur.fetch_pandas_all()
|
||||||
|
df.to_pickle(cache_pkl_path)
|
||||||
|
return df
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
## Python Kernel Management
|
## Python Kernel Management
|
||||||
```
|
```
|
||||||
# install new kernel "my_new_env"
|
# install new kernel "my_new_env"
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue