lnd-postgres-kv-decoder.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # lnd-postgres-kv-decoder.py | |
| fromdecoupleimportconfig | |
| importpsycopg2 | |
| importpsycopg2.extras | |
| importpandasaspd | |
| importtime | |
| tables= ['channeldb_kv', 'decayedlogdb_kv', 'macaroondb_kv', | |
| 'towerclientdb_kv', 'towerserverdb_kv', 'walletdb_kv'] | |
| defbytea2bytes(value, cur): | |
| """ | |
| Typecaster for converting bytea data to bytes | |
| """ | |
| m=psycopg2.BINARY(value, cur) | |
| ifmisnotNone: | |
| returnm.tobytes() | |
| # register the BYTE2BYTES type for psycopg2 | |
| BYTEA2BYTES=psycopg2.extensions.new_type(psycopg2.BINARY.values, 'BYTEA2BYTES', bytea2bytes) | |
| psycopg2.extensions.register_type(BYTEA2BYTES) | |
| deflnddb_conn(): | |
| """ | |
| Connect to Postgres Database using psycopg2 | |
| - credentials should be stored in an .env file for python-decouple to read from | |
| """ | |
| dbname=config('DB_NAME_LND') | |
| user=config('DB_USER_LND') | |
| password=config('DB_PASSWORD_LND') | |
| host=config('DB_HOST_LND') | |
| port=config('DB_PORT_LND', cast=int) | |
| try: | |
| conn=psycopg2.connect(dbname=dbname, | |
| user=user, | |
| password=password, | |
| host=host, | |
| port=port, | |
| sslmode='require' | |
| ) | |
| except (Exception, psycopg2.Error) ase: | |
| print(f"Error while connecting to DB:\n", e) | |
| conn=None | |
| returnconn | |
| defselect_all(conn, table_name): | |
| """ | |
| Select all records from a given table in the Postgres database | |
| """ | |
| dict_cur=conn.cursor(cursor_factory=psycopg2.extras.DictCursor) | |
| dict_cur.execute(f"select key::bytea, value::bytea, parent_id, id, sequence from {table_name}" | |
| "where value is not null "); | |
| rec=dict_cur.fetchall() | |
| dict_cur.close() | |
| returnrec | |
| defsingle_table(table_name): | |
| """ | |
| Select all records from a single table and read them into a Pandas dataframe | |
| """ | |
| conn=lnddb_conn() | |
| t=table_name | |
| rec=select_all(conn, t) | |
| col_names= ['key', 'value', 'parent_id', 'id', 'sequence'] | |
| df=pd.DataFrame([[row[col] forcolincol_names] forrowinrec], columns=col_names) | |
| conn.close() | |
| returndf | |
| defis_alpha_dash(w): | |
| """ | |
| Check whether a given value can be decoded into an alpha string including dashes | |
| """ | |
| ifnotisinstance(w, str): | |
| try: | |
| w=w.decode("utf-8") | |
| except: | |
| pass | |
| alphanumdash= ['a','b','c','d','e','f','g','h','i', | |
| 'j','k','l','m','n','o','p','q','r', | |
| 's','t','u','v','w','x','y','z','-'] | |
| chars= [] | |
| forcinw: | |
| ifcinalphanumdash: | |
| chars.append(c) | |
| chars_w=''.join(chars) | |
| iflen(chars_w) ==len(w): | |
| returnTrue, chars_w | |
| else: | |
| returnFalse, w | |
| defis_alphanum_dash(w): | |
| """ | |
| Check whether a given value can be decoded into an alphanumeric string including dashes | |
| """ | |
| ifnotisinstance(w, str): | |
| try: | |
| w=w.decode("utf-8") | |
| except: | |
| pass | |
| alphanumdash= ['a','b','c','d','e','f','g','h','i', | |
| 'j','k','l','m','n','o','p','q','r', | |
| 's','t','u','v','w','x','y','z','0', | |
| '1','2','3','4','5','6','7','8','9', | |
| '-'] | |
| chars= [] | |
| forcinw: | |
| ifcinalphanumdash: | |
| chars.append(c) | |
| chars_w=''.join(chars) | |
| iflen(chars_w) ==len(w): | |
| returnTrue, chars_w | |
| else: | |
| returnFalse, w | |
| defmain(): | |
| """ | |
| Read data from all tables and print out human-readable, decoded keys and values | |
| """ | |
| conn=lnddb_conn() | |
| fortintables: | |
| print(f"fetching data from {t}...") | |
| rec=select_all(conn, t) | |
| col_names= ['key', 'value', 'parent_id', 'id', 'sequence'] | |
| df=pd.DataFrame([[row[col] forcolincol_names] forrowinrec], columns=col_names) | |
| column_names= ['key', 'value'] | |
| forcolincolumn_names: | |
| unique_entries= [] | |
| forkindf[col]: | |
| ifk!=Noneandlen(k) >0: | |
| alphanum_dash, k2=is_alphanum_dash(k) | |
| ifalphanum_dash: | |
| alpha_dash, k3=is_alpha_dash(k2) | |
| ifalpha_dash: | |
| iflen(k3) >0: | |
| ifk3notinunique_entries: | |
| unique_entries.append(k3) | |
| else: | |
| k4=binascii.hexlify(k).decode('utf-8') | |
| alphanum_dahsh, k5=is_alphanum_dash(k4) | |
| ifalphanum_dash: | |
| alpha_dash, k6=is_alpha_dash(k5) | |
| ifalpha_dash: | |
| iflen(k6) >0: | |
| ifk6notinunique_entries: | |
| unique_entries.append(k6) | |
| print(f"\nUnique Names Decoded from '{column_name}' Column in '{t}' Table:\n") | |
| foriinunique_entries: | |
| print(i) | |
| print() | |
| print("-"*80) | |
| print() | |
| time.sleep(2) | |
| conn.close() | |
| print() | |
| print("DONE") | |
| if__name__=='__main__': | |
| main() |
