Csv2Table Column Type Update (#5631) (7323599d) · Commits · aflab / astrogeology / ISIS3

CHANGELOG.md

+1 −0

Original line number	Diff line number	Diff line
		@@ -53,6 +53,7 @@ release.
		- Added majority replacement for reduce app [#5101](https://github.com/DOI-USGS/ISIS3/issues/5101).
		- Added HRSC support in socetlinescankeywords [#5465](https://github.com/DOI-USGS/ISIS3/issues/5465)
		- Added option to save and apply bundle adjustment values in `jigsaw` [#4474](https://github.com/DOI-USGS/ISIS3/issues/4474)
		- Added the ability to pass column types in csv2table to set the column types in the resulting ISIS table [#5631](https://github.com/DOI-USGS/ISIS3/pull/5631)

		### Changed
		- Refactored the pixel2map app

isis/src/base/apps/csv2table/csv2table.cpp

0 → 100644

+218 −0

Original line number	Diff line number	Diff line
		/** This is free and unencumbered software released into the public domain.

		The authors of ISIS do not claim copyright on the contents of this file.
		For more details about the LICENSE terms and the AUTHORS, you will
		find files of those names at the top level of this repository. **/

		/* SPDX-License-Identifier: CC0-1.0 */
		#include "csv2table.h"

		#include <vector>

		#include <QRegularExpression>
		#include <QString>

		#include "Cube.h"
		#include "CSVReader.h"
		#include "IException.h"
		#include "IString.h"
		#include "Pvl.h"
		#include "PvlObject.h"
		#include "Table.h"
		#include "TableField.h"
		#include "TableRecord.h"

		using namespace std;

		namespace Isis {
		/**
		* csminit a cube in an Application
		*
		* @param ui The Application UI
		* @param(out) log The Pvl that attempted models will be logged to
		*/
		void csv2table(UserInterface &ui, Pvl *log) {
		// Read the CSV file and get the header
		QString csvFileName = ui.GetFileName("csv");
		CSVReader reader;
		try {
		reader = CSVReader(csvFileName, true);
		}
		catch(IException &e) {
		QString msg = "Failed to read CSV file [" + csvFileName + "].";
		throw IException(e, IException::Io, msg, _FILEINFO_);
		}
		int numColumns = reader.columns();
		int numRows = reader.rows();
		if (numColumns < 1 \|\| numRows < 1) {
		QString msg = "CSV file does not have data.\nFile has [" + toString(numRows) +
		"] rows and [" + toString(numColumns) +"] columns.";
		throw IException(IException::User, msg, _FILEINFO_);
		}

		std::vector<QString> fieldTypes;
		ui.GetAsString("coltypes", fieldTypes);

		std::vector<TableField::Type> tableTypes;
		if (fieldTypes.size() == 1 && fieldTypes[0] == "") {
		for (int i = 0; i < numColumns; i++) {
		tableTypes.push_back(TableField::Double);
		}
		}
		else {
		if (fieldTypes.size() == numColumns) {
		for (QString type: fieldTypes) {
		QString upper_type = type.toUpper();
		if (upper_type == "INTEGER") {
		tableTypes.push_back(TableField::Type::Integer);
		}
		else if (upper_type == "DOUBLE") {
		tableTypes.push_back(TableField::Type::Double);
		}
		else if (upper_type == "TEXT") {
		tableTypes.push_back(TableField::Type::Text);
		}
		else if (upper_type == "REAL") {
		tableTypes.push_back(TableField::Type::Real);
		}
		else {
		QString msg = "Field [" + type + "] cannot be translated. Accepted types are "
		"Integer, Double, Text, and Real";
		throw IException(IException::User, msg, _FILEINFO_);
		}
		}
		}
		else {
		int numFields = fieldTypes.size();
		QString msg = "Number of fields provided does not equal the number of columns in the CSV. "
		"Number of fields [" + toString(numFields) +
		"] vs Number of Columns [" + toString(numColumns) + "]";
		throw IException(IException::User, msg, _FILEINFO_);
		}
		}


		CSVReader::CSVAxis header = reader.getHeader();

		// Construct an empty table with the CSV header as field names
		// Collect identical field names together, including those with (###) at the end, so a single
		// table field with multiple values can be created.
		TableRecord tableRow;
		QRegularExpression rex(R"((?<name>\w+)($(?<index>[0-9]*)$\|))");
		for (int columnIndex = 0; columnIndex < numColumns; columnIndex++) {
		QRegularExpressionMatch match = rex.match(header[columnIndex]);
		if (match.hasMatch()) {
		QString name = match.captured("name");
		QString index = match.captured("index");

		// If the next column header is different, create a field for this one
		QRegularExpressionMatch nextMatch = (columnIndex<numColumns-1)?rex.match(header[columnIndex+1]):QRegularExpressionMatch();
		if ((columnIndex == numColumns-1) \|\| (nextMatch.hasMatch() && (name != nextMatch.captured("name")))) {
		TableField columnField(name, tableTypes[columnIndex], (index.length()>0)?(index.toInt()+1):1);
		tableRow += columnField;
		}
		}
		}

		QString tableName = ui.GetString("tablename");
		Table table(tableName, tableRow);

		// Fill the table from the csv
		for (int rowIndex = 0; rowIndex < numRows; rowIndex++) {
		CSVReader::CSVAxis csvRow = reader.getRow(rowIndex);
		for (int columnIndex = 0, fieldIndex = 0; columnIndex < numColumns; ) {
		if (tableRow[fieldIndex].size() == 1 \|\|
		tableRow[fieldIndex].isText()) {
		switch(tableTypes[columnIndex]) {
		case TableField::Type::Integer:
		tableRow[fieldIndex] = toInt(csvRow[columnIndex++]);
		break;
		case TableField::Type::Double:
		tableRow[fieldIndex] = toDouble(csvRow[columnIndex++]);
		break;
		case TableField::Type::Text:
		tableRow[fieldIndex] = QString(csvRow[columnIndex++]);
		break;
		case TableField::Type::Real:
		tableRow[fieldIndex] = (float)toDouble(csvRow[columnIndex++]);
		break;
		}
		}
		else {
		std::vector<int> intVector;
		std::vector<double> dblVector;
		std::vector<float> realVector;
		QString strMsg = "TableRecord can't handle list of Strings";
		switch(tableTypes[columnIndex]) {
		case TableField::Type::Integer:
		for (int arrayLen = 0; arrayLen < tableRow[fieldIndex].size(); arrayLen++) {
		intVector.push_back(toInt(csvRow[columnIndex++]));
		}
		tableRow[fieldIndex] = intVector;
		break;
		case TableField::Type::Double:
		for (int arrayLen = 0; arrayLen < tableRow[fieldIndex].size(); arrayLen++) {
		dblVector.push_back(toDouble(csvRow[columnIndex++]));
		}
		tableRow[fieldIndex] = dblVector;
		break;
		case TableField::Type::Text:
		throw IException(IException::User, strMsg, _FILEINFO_);
		break;
		case TableField::Type::Real:
		for (int arrayLen = 0; arrayLen < tableRow[fieldIndex].size(); arrayLen++) {
		realVector.push_back((float)toDouble(csvRow[columnIndex++]));
		}
		tableRow[fieldIndex] = realVector;
		break;
		}
		intVector.clear();
		dblVector.clear();
		realVector.clear();
		}
		fieldIndex++;
		}
		table += tableRow;
		}

		// If a set of additional label keywords was given then add them to the table's pvl description
		if (ui.WasEntered("label")) {
		QString labelPvlFilename = ui.GetFileName("label");
		Pvl labelPvl;
		try {
		labelPvl.read(labelPvlFilename);
		}
		catch(IException &e) {
		QString msg = "Failed to read PVL label file [" + labelPvlFilename + "].";
		throw IException(e, IException::Io, msg, _FILEINFO_);
		}

		PvlObject &tableLabel = table.Label();
		for (int keyIndex = 0; keyIndex < labelPvl.keywords(); keyIndex++) {
		tableLabel.addKeyword(labelPvl[keyIndex]);
		}
		}

		// Write the table to the cube
		QString outCubeFileName(ui.GetCubeName("to"));
		Cube outCube;
		try {
		outCube.open(outCubeFileName, "rw");
		}
		catch(IException &e) {
		QString msg = "Could not open output cube [" + outCubeFileName + "].";
		throw IException(e, IException::Io, msg, _FILEINFO_);
		}

		try {
		outCube.write(table);
		}
		catch(IException &e) {
		QString msg = "Could not write output table [" + tableName +
		"] to output cube [" + outCubeFileName + "].";
		throw IException(e, IException::Io, msg, _FILEINFO_);
		}

		outCube.close();
		}
		}
		No newline at end of file

isis/src/base/apps/csv2table/csv2table.h

0 → 100644

+18 −0

Original line number	Diff line number	Diff line
		#ifndef csv2table_h
		#define csv2table_h
		/** This is free and unencumbered software released into the public domain.

		The authors of ISIS do not claim copyright on the contents of this file.
		For more details about the LICENSE terms and the AUTHORS, you will
		find files of those names at the top level of this repository. **/

		/* SPDX-License-Identifier: CC0-1.0 */

		#include "Pvl.h"
		#include "UserInterface.h"

		namespace Isis {
		extern void csv2table(UserInterface &ui, Pvl *log=nullptr);
		}

		#endif
		No newline at end of file

isis/src/base/apps/csv2table/csv2table.xml

+15 −0

Original line number	Diff line number	Diff line
		@@ -39,6 +39,9 @@
		Added ability to convert CSV files with indicies into table field arrays
		instead of individual table fields.
		</change>
		<change name="Adam Paquette" date="2024-10-08">
		Added the ability to pass types for each column in the CSV
		</change>
		</history>

		<groups>
		@@ -94,6 +97,18 @@
		the cube it will be overwritten.
		</description>
		</parameter>

		<parameter name = "coltypes">
		<type>string</type>
		<brief>Column types of the table</brief>
		<description>
		ISIS command line list of Tablefield types. Specified as such, '(type1, type2)'
		where the allowed types are "Double", "Integer", "Float", "Text".
		</description>
		<default>
		<item></item>
		</default>
		</parameter>
		</group>

		</groups>

isis/src/base/apps/csv2table/main.cpp

+12 −135

Original line number	Diff line number	Diff line
		/**
		* @file
		* $Revision: 1.8 $
		* $Date: 2010/04/08 15:28:20 $
		*
		* Unless noted otherwise, the portions of Isis written by the USGS are public
		* domain. See individual third-party library and package descriptions for
		* intellectual property information,user agreements, and related information.
		*
		* Although Isis has been used by the USGS, no warranty, expressed or implied,
		* is made by the USGS as to the accuracy and functioning of such software
		* and related material nor shall the fact of distribution constitute any such
		* warranty, and no responsibility is assumed by the USGS in connection
		* therewith.
		*
		* For additional information, launch
		* $ISISROOT/doc//documents/Disclaimers/Disclaimers.html in a browser or see
		* the Privacy & Disclaimers page on the Isis website,
		* http://isis.astrogeology.usgs.gov, and the USGS privacy and disclaimers on
		* http://www.usgs.gov/privacy.html.
		*/
		/** This is free and unencumbered software released into the public domain.

		#include "Isis.h"
		The authors of ISIS do not claim copyright on the contents of this file.
		For more details about the LICENSE terms and the AUTHORS, you will
		find files of those names at the top level of this repository. **/

		#include <vector>
		/* SPDX-License-Identifier: CC0-1.0 */
		#include "Isis.h"

		#include <QRegularExpression>
		#include <QString>
		#include "csv2table.h"

		#include "Cube.h"
		#include "CSVReader.h"
		#include "Application.h"
		#include "IException.h"
		#include "IString.h"
		#include "Pvl.h"
		#include "PvlObject.h"
		#include "Table.h"
		#include "TableField.h"
		#include "TableRecord.h"

		using namespace std;
		using namespace Isis;


		void IsisMain() {
		UserInterface &ui = Application::GetUserInterface();

		// Read the CSV file and get the header
		QString csvFileName = ui.GetFileName("csv");
		CSVReader reader;
		try {
		reader = CSVReader(csvFileName, true);
		}
		catch(IException &e) {
		QString msg = "Failed to read CSV file [" + csvFileName + "].";
		throw IException(e, IException::Io, msg, _FILEINFO_);
		}
		int numColumns = reader.columns();
		int numRows = reader.rows();
		if (numColumns < 1 \|\| numRows < 1) {
		QString msg = "CSV file does not have data.\nFile has [" + toString(numRows) +
		"] rows and [" + toString(numColumns) +"] columns.";
		throw IException(IException::User, msg, _FILEINFO_);
		}
		CSVReader::CSVAxis header = reader.getHeader();

		// Construct an empty table with the CSV header as field names
		// Collect identical field names together, including those with (###) at the end, so a single
		// table field with multiple values can be created.
		TableRecord tableRow;
		QRegularExpression rex(R"((?<name>\w+)($(?<index>[0-9]*)$\|))");
		for (int columnIndex = 0; columnIndex < numColumns; columnIndex++) {
		QRegularExpressionMatch match = rex.match(header[columnIndex]);
		if (match.hasMatch()) {
		QString name = match.captured("name");
		QString index = match.captured("index");

		// If the next column header is different, create a field for this one
		QRegularExpressionMatch nextMatch = (columnIndex<numColumns-1)?rex.match(header[columnIndex+1]):QRegularExpressionMatch();
		if ((columnIndex == numColumns-1) \|\| (nextMatch.hasMatch() && (name != nextMatch.captured("name")))) {
		TableField columnField(name, TableField::Double, (index.length()>0)?(index.toInt()+1):1);
		tableRow += columnField;
		}
		}
		}

		QString tableName = ui.GetString("tablename");
		Table table(tableName, tableRow);

		// Fill the table from the csv
		for (int rowIndex = 0; rowIndex < numRows; rowIndex++) {
		CSVReader::CSVAxis csvRow = reader.getRow(rowIndex);
		for (int columnIndex = 0, fieldIndex = 0; columnIndex < numColumns; ) {
		if (tableRow[fieldIndex].size() == 1) {
		tableRow[fieldIndex] = toDouble(csvRow[columnIndex++]);
		}
		else {
		std::vector<double> dblVector;
		for (int arrayLen = 0; arrayLen < tableRow[fieldIndex].size(); arrayLen++) {
		dblVector.push_back(toDouble(csvRow[columnIndex++]));
		}
		tableRow[fieldIndex] = dblVector;
		}
		fieldIndex++;
		}
		table += tableRow;
		}

		// If a set of additional label keywords was given then add them to the table's pvl description
		if (ui.WasEntered("label")) {
		QString labelPvlFilename = ui.GetFileName("label");
		Pvl labelPvl;
		try {
		labelPvl.read(labelPvlFilename);
		}
		catch(IException &e) {
		QString msg = "Failed to read PVL label file [" + labelPvlFilename + "].";
		throw IException(e, IException::Io, msg, _FILEINFO_);
		}

		PvlObject &tableLabel = table.Label();
		for (int keyIndex = 0; keyIndex < labelPvl.keywords(); keyIndex++) {
		tableLabel.addKeyword(labelPvl[keyIndex]);
		}
		}

		// Write the table to the cube
		QString outCubeFileName(ui.GetCubeName("to"));
		Cube outCube;
		try {
		outCube.open(outCubeFileName, "rw");
		}
		catch(IException &e) {
		QString msg = "Could not open output cube [" + outCubeFileName + "].";
		throw IException(e, IException::Io, msg, _FILEINFO_);
		}

		try {
		outCube.write(table);
		}
		catch(IException &e) {
		QString msg = "Could not write output table [" + tableName +
		"] to output cube [" + outCubeFileName + "].";
		throw IException(e, IException::Io, msg, _FILEINFO_);
		}

		outCube.close();
		Pvl appLog;
		csv2table(ui, &appLog);
		}
		No newline at end of file