• R/O
  • SSH

execsql: Commit

Default repository for execsql.py


Commit MetaInfo

Révisionbc651354a0e467884d11eb6a4d4b1a5f0e2b788d (tree)
l'heure2022-06-19 08:27:48
Auteurrdnielsen
Commiterrdnielsen

Message de Log

Added code to import Feather data files, but it is not functional with all standard Postgres datatypes, and is not documented.

Change Summary

Modification

diff -r 3c58f65080a8 -r bc651354a0e4 doc/source/requirements.rst
--- a/doc/source/requirements.rst Sat Jun 18 13:42:03 2022 -0700
+++ b/doc/source/requirements.rst Sat Jun 18 16:27:48 2022 -0700
@@ -64,7 +64,7 @@
6464
6565 If data are to be :ref:`exported <export>` to the feather file format,
6666 the `pandas <https://pypi.org/project/pandas/>`_ and
67-`feather-format <https://pypi.org/project/feather-format/>`_
67+`pyarrow <https://pypi.org/project/pyarrow/>`_
6868 Python packages must also be installed.
6969
7070 If data are to be :ref:`exported <export>` to the HDF5 file format,
diff -r 3c58f65080a8 -r bc651354a0e4 execsql/execsql.py
--- a/execsql/execsql.py Sat Jun 18 13:42:03 2022 -0700
+++ b/execsql/execsql.py Sat Jun 18 16:27:48 2022 -0700
@@ -8583,6 +8583,36 @@
85838583 x_import_parquet)
85848584
85858585
8586+#**** IMPORT FROM FEATHER
8587+def x_import_feather(**kwargs):
8588+ # is_new should have values of 0, 1, or 2
8589+ newstr = kwargs['new']
8590+ if newstr:
8591+ is_new = 1 + ['new', 'replacement'].index(newstr.lower())
8592+ else:
8593+ is_new = 0
8594+ schemaname = kwargs['schema']
8595+ tablename = kwargs['table']
8596+ filename = kwargs['filename']
8597+ if len(filename) > 1 and filename[0] == "~" and filename[1] == os.sep:
8598+ filename = os.path.join(os.path.expanduser(r'~'), filename[2:])
8599+ if not os.path.exists(filename):
8600+ raise ErrInfo(type="cmd", command_text=kwargs['metacommandline'], other_msg='Input file %s does not exist' % filename)
8601+ sz, dt = file_size_date(filename)
8602+ exec_log.log_status_info(u"IMPORTing from Feather file %s (%s, %s)" % (filename, sz, dt))
8603+ try:
8604+ import_feather(dbs.current(), schemaname, tablename, filename, is_new)
8605+ except ErrInfo:
8606+ raise
8607+ except:
8608+ raise ErrInfo("exception", exception_msg=exception_desc(), other_msg="Can't import data from Feather data file %s" % filename)
8609+ return None
8610+
8611+metacommandlist.add(
8612+ ins_table_rxs(r'^\s*IMPORT\s+TO\s+(?:(?P<new>NEW|REPLACEMENT)\s+)?', ins_fn_rxs(r'\s+FROM\s+FEATHER\s+', r'\s*$')),
8613+ x_import_feather)
8614+
8615+
85868616 #**** DEBUG WRITE ODBC_DRIVERS
85878617 def x_debug_write_odbc_drivers(**kwargs):
85888618 try:
@@ -13028,17 +13058,11 @@
1302813058 def write_query_to_feather(outfile, headers, rows):
1302913059 try:
1303013060 import pandas as pd
13031- import feather
13061+ import pyarrow.feather
1303213062 except:
13033- raise ErrInfo("exception", exception_msg=exception_desc(), other_msg=u"The pandas and feather Python libraries must be installed to export data to the feather format.")
13034- data = []
13035- for row in rows:
13036- txtrow = []
13037- for val in row:
13038- txtrow.append(type(u"")(val))
13039- data.append(txtrow)
13040- df = pd.DataFrame(data, columns=headers)
13041- feather.write_dataframe(df, outfile)
13063+ raise ErrInfo("exception", exception_msg=exception_desc(), other_msg=u"The pandas and pyarrow Python packages must be installed to export data to the feather format.")
13064+ df = pd.DataFrame(rows, columns=headers)
13065+ pyarrow.feather.write_feather(df, outfile)
1304213066
1304313067
1304413068 def pause(message=None, max_time=None):
@@ -13828,6 +13852,19 @@
1382813852 hdrs, data = xls_data(filename, sheetname, junk_header_rows, encoding)
1382913853 import_data_table(db, schemaname, tablename, is_new, hdrs, data)
1383013854
13855+def import_feather(db, schemaname, tablename, filename, is_new):
13856+ try:
13857+ import pandas as pd
13858+ import pyarrow.feather
13859+ except:
13860+ raise ErrInfo("exception", exception_msg=exception_desc(), other_msg=u"The pandas and pyarrow Python libraries must be installed to import data from the Feather format.")
13861+ #df = pyarrow.feather.read_feather(filename)
13862+ df = pd.read_feather(filename)
13863+ hdrs = df.columns.values.tolist()
13864+ data = df.values.tolist()
13865+ import_data_table(db, schemaname, tablename, is_new, hdrs, data)
13866+
13867+
1383113868 def import_parquet(db, schemaname, tablename, filename, is_new):
1383213869 try:
1383313870 import pandas as pd
Afficher sur ancien navigateur de dépôt.