/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.sysds.runtime.util;

import java.io.IOException;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map.Entry;
import java.util.StringTokenizer;

import org.apache.commons.lang3.StringUtils;
import org.apache.commons.math3.linear.Array2DRowRealMatrix;
import org.apache.commons.math3.linear.BlockRealMatrix;
import org.apache.commons.math3.linear.RealMatrix;
import org.apache.sysds.common.Types;
import org.apache.sysds.common.Types.FileFormat;
import org.apache.sysds.common.Types.ValueType;
import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
import org.apache.sysds.runtime.controlprogram.caching.FrameObject;
import org.apache.sysds.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysds.runtime.controlprogram.caching.TensorObject;
import org.apache.sysds.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysds.runtime.data.BasicTensorBlock;
import org.apache.sysds.runtime.data.DataTensorBlock;
import org.apache.sysds.runtime.data.DenseBlock;
import org.apache.sysds.runtime.data.DenseBlockFactory;
import org.apache.sysds.runtime.data.SparseBlock;
import org.apache.sysds.runtime.data.TensorBlock;
import org.apache.sysds.runtime.frame.data.FrameBlock;
import org.apache.sysds.runtime.frame.data.iterators.IteratorFactory;
import org.apache.sysds.runtime.frame.data.lib.FrameFromMatrixBlock;
import org.apache.sysds.runtime.frame.data.lib.MatrixBlockFromFrame;
import org.apache.sysds.runtime.instructions.cp.BooleanObject;
import org.apache.sysds.runtime.instructions.cp.CPOperand;
import org.apache.sysds.runtime.instructions.cp.Data;
import org.apache.sysds.runtime.instructions.cp.ListObject;
import org.apache.sysds.runtime.instructions.cp.ScalarObject;
import org.apache.sysds.runtime.io.FileFormatProperties;
import org.apache.sysds.runtime.io.MatrixReader;
import org.apache.sysds.runtime.io.MatrixReaderFactory;
import org.apache.sysds.runtime.io.MatrixWriter;
import org.apache.sysds.runtime.io.MatrixWriterFactory;
import org.apache.sysds.runtime.io.ReadProperties;
import org.apache.sysds.runtime.io.TensorReader;
import org.apache.sysds.runtime.io.TensorReaderFactory;
import org.apache.sysds.runtime.io.TensorWriter;
import org.apache.sysds.runtime.io.TensorWriterFactory;
import org.apache.sysds.runtime.matrix.data.CTableMap;
import org.apache.sysds.runtime.matrix.data.IJV;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
import org.apache.sysds.runtime.matrix.data.MatrixIndexes;
import org.apache.sysds.runtime.meta.DataCharacteristics;


/**
 * This class provides methods to read and write matrix blocks from to HDFS using different data formats.
 * Those functionalities are used especially for CP read/write and exporting in-memory matrices to HDFS
 * (before executing MR jobs).
 */
public class DataConverter {
	private static final String DELIM = " ";

	//////////////
	// READING and WRITING of matrix blocks to/from HDFS
	// (textcell, binarycell, binaryblock)
	///////

	public static void writeMatrixToHDFS(MatrixBlock mat, String dir, FileFormat fmt,  DataCharacteristics dc )
		throws IOException {
		writeMatrixToHDFS(mat, dir, fmt, dc, -1, null);
	}

	public static void writeMatrixToHDFS(MatrixBlock mat, String dir, FileFormat fmt, DataCharacteristics dc, int replication, FileFormatProperties formatProperties)
		throws IOException {
		writeMatrixToHDFS(mat, dir, fmt, dc, -1, null, false);
	}

	public static void writeMatrixToHDFS(MatrixBlock mat, String dir, FileFormat fmt, DataCharacteristics dc, int replication, FileFormatProperties formatProperties, boolean diag)
		throws IOException {
		MatrixWriter writer = MatrixWriterFactory.createMatrixWriter( fmt, replication, formatProperties );
		writer.writeMatrixToHDFS(mat, dir, dc.getRows(), dc.getCols(), dc.getBlocksize(), dc.getNonZeros(), diag);
	}

	public static void writeTensorToHDFS(TensorBlock tensor, String dir, FileFormat fmt, DataCharacteristics dc)
		throws IOException {
		TensorWriter writer = TensorWriterFactory.createTensorWriter(fmt);
		int blen = dc.getBlocksize();
		writer.writeTensorToHDFS(tensor, dir, blen);
	}


	public static MatrixBlock readMatrixFromHDFS(String dir, FileFormat fmt, long rlen, long clen, int blen)
		throws IOException
	{
		ReadProperties prop = new ReadProperties();

		prop.path = dir;
		prop.fmt = fmt;
		prop.rlen = rlen;
		prop.clen = clen;
		prop.blen = blen;

		return readMatrixFromHDFS(prop);
	}

	public static MatrixBlock readMatrixFromHDFS(String dir, FileFormat fmt, long rlen, long clen, int blen, long expectedNnz)
		throws IOException
	{
		ReadProperties prop = new ReadProperties();

		prop.path = dir;
		prop.fmt = fmt;
		prop.rlen = rlen;
		prop.clen = clen;
		prop.blen = blen;
		prop.expectedNnz = expectedNnz;

		return readMatrixFromHDFS(prop);
	}

	public static MatrixBlock readMatrixFromHDFS(String dir, FileFormat fmt, long rlen, long clen,
		int blen, long expectedNnz, FileFormatProperties formatProperties)
		throws IOException
	{
		ReadProperties prop = new ReadProperties();

		prop.path = dir;
		prop.fmt = fmt;
		prop.rlen = rlen;
		prop.clen = clen;
		prop.blen = blen;
		prop.expectedNnz = expectedNnz;
		prop.formatProperties = formatProperties;

		return readMatrixFromHDFS(prop);
	}

	public static TensorBlock readTensorFromHDFS(String dir, FileFormat fmt, long[] dims, int blen,
		ValueType[] schema) throws IOException {
		TensorBlock ret;
		try {
			TensorReader reader = TensorReaderFactory.createTensorReader(fmt);
			ret = reader.readTensorFromHDFS(dir, dims, blen, schema);
		}
		catch(DMLRuntimeException rex)
		{
			throw new IOException(rex);
		}
		return ret;
	}

	/**
	 * Core method for reading matrices in format textcell, matrixmarket, binarycell, or binaryblock 
	 * from HDFS into main memory. For expected dense matrices we directly copy value- or block-at-a-time 
	 * into the target matrix. In contrast, for sparse matrices, we append (column-value)-pairs and do a 
	 * final sort if required in order to prevent large reorg overheads and increased memory consumption 
	 * in case of unordered inputs.  
	 *
	 * DENSE MxN input:
	 *  * best/average/worst: O(M*N)
	 * SPARSE MxN input
	 *  * best (ordered, or binary block w/ clen&lt;=blen): O(M*N)
	 *  * average (unordered): O(M*N*log(N))
	 *  * worst (descending order per row): O(M * N^2)
	 *
	 * NOTE: providing an exact estimate of 'expected sparsity' can prevent a full copy of the result
	 * matrix block (required for changing sparse-&gt;dense, or vice versa)
	 *
	 * @param prop read properties
	 * @return matrix block
	 * @throws IOException if IOException occurs
	 */
	public static MatrixBlock readMatrixFromHDFS(ReadProperties prop)
		throws IOException
	{
		//Timing time = new Timing(true);

		//core matrix reading
		MatrixBlock ret = null;
		try {
			MatrixReader reader = MatrixReaderFactory.createMatrixReader(prop);
			ret = reader.readMatrixFromHDFS(prop.path, prop.rlen, prop.clen, prop.blen, prop.expectedNnz);
		}
		catch(DMLRuntimeException rex)
		{
			throw new IOException(rex);
		}

		//System.out.println("read matrix ("+prop.rlen+","+prop.clen+","+ret.getNonZeros()+") in "+time.stop());

		return ret;
	}


	//////////////
	// Utils for CREATING and COPYING matrix blocks
	///////

	/**
	 * Creates a two-dimensional double matrix of the input matrix block.
	 *
	 * @param mb matrix block
	 * @return 2d double array
	 */
	public static double[][] convertToDoubleMatrix( MatrixBlock mb ) {
		int rows = mb.getNumRows();
		int cols = mb.getNumColumns();
		double[][] ret = new double[rows][cols]; //0-initialized
		if(mb instanceof CompressedMatrixBlock){
			mb = ((CompressedMatrixBlock)mb).getUncompressed("convert to Double Matrix");
		}
		if( mb.getNonZeros() > 0 ) {
			if( mb.isInSparseFormat() ) {
				Iterator<IJV> iter = mb.getSparseBlockIterator();
				while( iter.hasNext() ) {
					IJV cell = iter.next();
					ret[cell.getI()][cell.getJ()] = cell.getV();
				}
			}
			else {
				double[] a = mb.getDenseBlockValues();
				for( int i=0, ix=0; i<rows; i++ )
					for( int j=0; j<cols; j++, ix++ )
						ret[i][j] = a[ix];
			}
		}
		return ret;
	}

	public static boolean [] convertToBooleanVector(MatrixBlock mb)
	{
		int rows = mb.getNumRows();
		int cols = mb.getNumColumns();
		boolean[] ret = new boolean[rows*cols]; //false-initialized

		if( mb.getNonZeros() > 0 )
		{
			if( mb.isInSparseFormat() ) {
				Iterator<IJV> iter = mb.getSparseBlockIterator();
				while( iter.hasNext() ) {
					IJV cell = iter.next();
					ret[cell.getI()*cols+cell.getJ()] = (cell.getV() != 0.0);
				}
			}
			else {
				DenseBlock d = mb.getDenseBlock();
				for( int i=0, cix=0; i<rows; i++ )
					for( int j=0; j<cols; j++, cix++)
						ret[cix] = (d.get(i, j) != 0.0);
			}
		}

		return ret;
	}

	public static int[] convertVectorToIndexList(MatrixBlock mb)
	{
		int rows = mb.getNumRows();
		int cols = mb.getNumColumns();

		if( mb.isEmpty() )
			return null;

		if( mb.isInSparseFormat() ) {
			if(rows == 1) {
				// row vector
				SparseBlock sb = mb.getSparseBlock();
				int[] tmp = sb.indexes(0);
				return (tmp.length == sb.size(0)) ? tmp :
					Arrays.copyOfRange(tmp, 0, sb.size(0));
			}
			else {
				// column vector
				int index = 0;
				int[] indices = new int[(int) mb.getNonZeros()];
				Iterator<IJV> iter = mb.getSparseBlockIterator();
				while(iter.hasNext()) {
					IJV cell = iter.next();
					if(cell.getV() != 0.0)
						indices[index++] = cell.getI() * cols + cell.getJ();
				}
				return indices;
			}
		}
		else {
			int[] indices = new int[(int) mb.getNonZeros()];
			DenseBlock d = mb.getDenseBlock();
			for(int i = 0, aix=0, cix=0; i < rows; i++)
				for(int j = 0; j < cols; j++, aix++)
					if(d.get(i, j) != 0.0)
						indices[cix++] = aix;
			return indices;
		}
	}

	public static int[] convertToIntVector( MatrixBlock mb) {
		int rows = mb.getNumRows();
		int cols = mb.getNumColumns();
		int[] ret = new int[rows*cols]; //0-initialized
		if( mb.isEmptyBlock(false) )
			return ret;
		if( mb.isInSparseFormat() ) {
			Iterator<IJV> iter = mb.getSparseBlockIterator();
			while( iter.hasNext() ) {
				IJV cell = iter.next();
				ret[cell.getI()*cols+cell.getJ()] = (int)cell.getV();
			}
		}
		else {
			//memcopy row major representation if at least 1 non-zero
			DenseBlock d = mb.getDenseBlock();
			for( int i=0, cix=0; i<rows; i++ )
				for( int j=0; j<cols; j++, cix++ )
					ret[cix] = (int)(d.get(i, j));
		}
		return ret;
	}

	public static long[] convertToLongVector( MatrixBlock mb) {
		int rows = mb.getNumRows();
		int cols = mb.getNumColumns();
		long[] ret = new long[rows*cols]; //0-initialized
		if( mb.isEmptyBlock(false) )
			return ret;
		if( mb.isInSparseFormat() ) {
			Iterator<IJV> iter = mb.getSparseBlockIterator();
			while( iter.hasNext() ) {
				IJV cell = iter.next();
				ret[cell.getI()*cols+cell.getJ()] = (int)cell.getV();
			}
		}
		else {
			//memcopy row major representation if at least 1 non-zero
			DenseBlock d = mb.getDenseBlock();
			for( int i=0, cix=0; i<rows; i++ )
				for( int j=0; j<cols; j++, cix++ )
					ret[cix] = (int)(d.get(i, j));
		}
		return ret;
	}

	public static DenseBlock convertToDenseBlock(MatrixBlock mb) {
		return convertToDenseBlock(mb, true);
	}

	public static DenseBlock convertToDenseBlock(MatrixBlock mb, boolean deep) {
		int rows = mb.getNumRows();
		int cols = mb.getNumColumns();
		DenseBlock ret = (!mb.isInSparseFormat() && mb.isAllocated() && !deep) ?
			mb.getDenseBlock() : DenseBlockFactory.createDenseBlock(rows, cols); //0-initialized

		if( !mb.isEmptyBlock(false) ) {
			if( mb.isInSparseFormat() ) {
				Iterator<IJV> iter = mb.getSparseBlockIterator();
				while( iter.hasNext() ) {
					IJV cell = iter.next();
					ret.set(cell.getI(), cell.getJ(), cell.getV());
				}
			}
			else if( deep ) {
				ret.set(mb.getDenseBlock());
			}
		}

		return ret;
	}

	public static double[] convertToDoubleVector(MatrixBlock mb) {
		return convertToDoubleVector(mb, true);
	}

	public static double[] convertToDoubleVector( MatrixBlock mb, boolean deep ) {
		return convertToDoubleVector(mb, deep, false);
	}

	public static double[] convertToDoubleVector( MatrixBlock mb, boolean deep, boolean allowNull )
	{
		if( mb.isEmpty() && allowNull )
			return null;

		int rows = mb.getNumRows();
		int cols = mb.getNumColumns();
		double[] ret = (!mb.isInSparseFormat() && mb.isAllocated() && !deep) ?
			mb.getDenseBlockValues() : new double[rows*cols]; //0-initialized

		if( !mb.isEmptyBlock(false) ) {
			if( mb.isInSparseFormat() ) {
				Iterator<IJV> iter = mb.getSparseBlockIterator();
				while( iter.hasNext() ) {
					IJV cell = iter.next();
					ret[cell.getI()*cols+cell.getJ()] = cell.getV();
				}
			}
			else if( deep ) {
				//memcopy row major representation if at least 1 non-zero
				System.arraycopy(mb.getDenseBlockValues(), 0, ret, 0, rows*cols);
			}
		}

		return ret;
	}

	public static List<Double> convertToDoubleList( MatrixBlock mb )
	{
		int rows = mb.getNumRows();
		int cols = mb.getNumColumns();
		long nnz = mb.getNonZeros();
		ArrayList<Double> ret = new ArrayList<>();

		if( mb.isInSparseFormat() )
		{
			Iterator<IJV> iter = mb.getSparseBlockIterator();
			while( iter.hasNext() ) {
				IJV cell = iter.next();
				ret.add( cell.getV() );
			}
			for( long i=nnz; i<(long)rows*cols; i++ )
				ret.add( 0d ); //add remaining values
		}
		else
		{
			DenseBlock d = mb.getDenseBlock();
			for( int i=0; i<rows; i++ )
				for( int j=0; j<cols; j++ )
					ret.add( d!=null?d.get(i, j) : 0 );
		}

		return ret;
	}

	/**
	 * Creates a dense Matrix Block and copies the given double matrix into it.
	 *
	 * @param data 2d double array
	 * @return matrix block
	 */
	public static MatrixBlock convertToMatrixBlock( double[][] data ) {
		int rows = data.length;
		int cols = (rows > 0)? data[0].length : 0;
		MatrixBlock mb = new MatrixBlock(rows, cols, false);
		try
		{
			//copy data to mb (can be used because we create a dense matrix)
			mb.init( data, rows, cols );
		}
		catch (Exception e){} //can never happen

		//check and convert internal representation
		mb.examSparsity();

		return mb;
	}

	/**
	 * Converts an Integer matrix to an MatrixBlock
	 *
	 * @param data Int matrix input that is converted to double MatrixBlock
	 * @return The matrixBlock constructed.
	 */
	public static MatrixBlock convertToMatrixBlock(int[][] data){
		int rows = data.length;
		int cols = (rows > 0)? data[0].length : 0;
		MatrixBlock res = new MatrixBlock(rows, cols, false);
		for(int row = 0; row< data.length; row++){
			for(int col = 0; col < cols; col++){
				double v = data[row][col];
				if( v != 0 )
					res.appendValue(row, col, v);
			}
		}
		return res;
	}

	/**
	 * Creates a dense Matrix Block and copies the given double vector into it.
	 *
	 * @param data double array
	 * @param columnVector if true, create matrix with single column. if false, create matrix with single row
	 * @return matrix block
	 */
	public static MatrixBlock convertToMatrixBlock( double[] data, boolean columnVector ) {
		int rows = columnVector ? data.length : 1;
		int cols = columnVector ? 1 : data.length;
		MatrixBlock mb = new MatrixBlock(rows, cols, false);
		//copy data to mb (can be used because we create a dense matrix)
		mb.init( data, rows, cols );
		mb.examSparsity();
		return mb;
	}

	public static MatrixBlock convertToMatrixBlock( HashMap<MatrixIndexes,Double> map )
	{
		// compute dimensions from the map
		long nrows=0, ncols=0;
		for (MatrixIndexes index : map.keySet()) {
			nrows = Math.max( nrows, index.getRowIndex() );
			ncols = Math.max( ncols, index.getColumnIndex() );
		}

		// convert to matrix block
		return convertToMatrixBlock(map, (int)nrows, (int)ncols);
	}

	/**
	 * NOTE: this method also ensures the specified matrix dimensions
	 *
	 * @param map map of matrix index keys and double values
	 * @param rlen number of rows
	 * @param clen number of columns
	 * @return matrix block
	 */
	public static MatrixBlock convertToMatrixBlock( HashMap<MatrixIndexes,Double> map, int rlen, int clen )
	{
		int nnz = map.size();
		boolean sparse = MatrixBlock.evalSparseFormatInMemory(rlen, clen, nnz);
		MatrixBlock mb = new MatrixBlock(rlen, clen, sparse, nnz);

		// copy map values into new block
		if( sparse ) //SPARSE <- cells
		{
			//append cells to sparse target (prevent shifting)
			for( Entry<MatrixIndexes,Double> e : map.entrySet() )
			{
				MatrixIndexes index = e.getKey();
				double value = e.getValue();
				int rix = (int)index.getRowIndex();
				int cix = (int)index.getColumnIndex();
				if( value != 0 && rix<=rlen && cix<=clen )
					mb.appendValue( rix-1, cix-1, value );
			}

			//sort sparse target representation
			mb.sortSparseRows();
		}
		else  //DENSE <- cells
		{
			//directly insert cells into dense target
			for( Entry<MatrixIndexes,Double> e : map.entrySet() )
			{
				MatrixIndexes index = e.getKey();
				double value = e.getValue();
				int rix = (int)index.getRowIndex();
				int cix = (int)index.getColumnIndex();
				if( value != 0 && rix<=rlen && cix<=clen )
					mb.set( rix-1, cix-1, value );
			}
		}

		return mb;
	}

	public static MatrixBlock convertToMatrixBlock( CTableMap map )
	{
		// compute dimensions from the map
		int nrows = (int)map.getMaxRow();
		int ncols = (int)map.getMaxColumn();

		// convert to matrix block
		return convertToMatrixBlock(map, nrows, ncols);
	}

	/**
	 * NOTE: this method also ensures the specified matrix dimensions
	 *
	 * @param map ?
	 * @param rlen number of rows
	 * @param clen number of columns
	 * @return matrix block
	 */
	public static MatrixBlock convertToMatrixBlock( CTableMap map, int rlen, int clen )
	{
		return map.toMatrixBlock(rlen, clen);
	}

	/**
	 * Converts a frame block with arbitrary schema into a matrix block.
	 * Since matrix block only supports value type double, we do a best
	 * effort conversion of non-double types which might result in errors
	 * for non-numerical data.
	 *
	 * @param frame frame block
	 * @return matrix block
	 */
	public static MatrixBlock convertToMatrixBlock(FrameBlock frame){
		return MatrixBlockFromFrame.convertToMatrixBlock(frame, 1);
	}

	/**
	 * Converts a frame block with arbitrary schema into a two dimensional
	 * string array.
	 *
	 * @param frame frame block
	 * @return 2d string array
	 */
	public static String[][] convertToStringFrame(FrameBlock frame)
	{
		String[][] ret = new String[frame.getNumRows()][];
		Iterator<String[]> iter = IteratorFactory.getStringRowIterator(frame);
		for( int i=0; iter.hasNext(); i++ ) {
			//deep copy output rows due to internal reuse
			ret[i] = iter.next().clone();
		}

		return ret;
	}

	/**
	 * Converts a two dimensions string array into a frame block of
	 * value type string. If the given array is null or of length 0,
	 * we return an empty frame block.
	 *
	 * @param data 2d string array
	 * @return frame block
	 */
	public static FrameBlock convertToFrameBlock(String[][] data) {
		//check for empty frame block
		if( data == null || data.length==0 )
			return new FrameBlock();

		//create schema and frame block
		ValueType[] schema = UtilFunctions.nCopies(data[0].length, ValueType.STRING);
		return convertToFrameBlock(data, schema);
	}

	public static FrameBlock convertToFrameBlock(String[][] data, ValueType[] schema) {
		//check for empty frame block
		if( data == null || data.length==0 )
			return new FrameBlock();

		//create frame block
		return new FrameBlock(schema, data);
	}

	public static FrameBlock convertToFrameBlock(String[][] data, ValueType[] schema, String[] colnames) {
		//check for empty frame block
		if( data == null || data.length==0 )
			return new FrameBlock();

		//create frame block
		return new FrameBlock(schema, colnames, data);
	}

	/**
	 * Converts a matrix block into a frame block of value type double.
	 *
	 * @param mb matrix block
	 * @return frame block of type double
	 */
	public static FrameBlock convertToFrameBlock(MatrixBlock mb) {
		return FrameFromMatrixBlock.convertToFrameBlock(mb, 1);
	}

	/**
	 * Converts a matrix block into a frame block of value type double.
	 *
	 * @param mb matrix block
	 * @param k  parallelization degree
	 * @return frame block of type double
	 */
	public static FrameBlock convertToFrameBlock(MatrixBlock mb, int k) {
		return FrameFromMatrixBlock.convertToFrameBlock(mb, k);
	}

	/**
	 * Converts a matrix block into a frame block of value type given.
	 * 
	 * @param mb matrix block
	 * @param vt value type target
	 * @return frame block of type given
	 */
	public static FrameBlock convertToFrameBlock(MatrixBlock mb, ValueType vt) {
		return FrameFromMatrixBlock.convertToFrameBlock(mb, vt, 1);
	}

	/**
	 * Converts a matrix block into a frame block of a given value type.
	 *
	 * @param mb matrix block
	 * @param vt value type
	 * @param k  parallelization degree
	 * @return a return frame block with the given schema
	 */
	public static FrameBlock convertToFrameBlock(MatrixBlock mb, ValueType vt, int k) {
		return FrameFromMatrixBlock.convertToFrameBlock(mb, vt, k);
	}

	/**
	 * Converts a matrix block into a frame block of with the given schema
	 * 
	 * @param mb     matrix block
	 * @param schema schema
	 * @return a return frame block with the given schema
	 */
	public static FrameBlock convertToFrameBlock(MatrixBlock mb, ValueType[] schema) {
		return FrameFromMatrixBlock.convertToFrameBlock(mb, schema, 1);
	}

	/**
	 * Converts a matrix block into a frame block of with the given schema
	 * 
	 * @param mb     matrix block
	 * @param schema schema
	 * @param k      parallelization degree
	 * @return a return frame block with the given schema
	 */
	public static FrameBlock convertToFrameBlock(MatrixBlock mb, ValueType[] schema, int k) {
		return FrameFromMatrixBlock.convertToFrameBlock(mb, schema, k);
	}

	public static TensorBlock convertToTensorBlock(MatrixBlock mb, ValueType vt, boolean toBasicTensor) {
		TensorBlock ret;
		if (toBasicTensor)
			ret = new TensorBlock(new BasicTensorBlock(vt, new int[] {mb.getNumRows(), mb.getNumColumns()}, false));
		else
			ret = new TensorBlock(new DataTensorBlock(vt, new int[] {mb.getNumRows(), mb.getNumColumns()}));
		ret.allocateBlock();
		if( mb.getNonZeros() > 0 ) {
			if( mb.isInSparseFormat() ) {
				Iterator<IJV> iter = mb.getSparseBlockIterator();
				while( iter.hasNext() ) {
					IJV cell = iter.next();
					ret.set(cell.getI(), cell.getJ(), cell.getV());
				}
			}
			else {
				double[] a = mb.getDenseBlockValues();
				for( int i=0, ix=0; i<mb.getNumRows(); i++ )
					for( int j=0; j<mb.getNumColumns(); j++, ix++ )
						ret.set(i, j, a[ix]);
			}
		}
		return ret;
	}

	public static MatrixBlock[] convertToMatrixBlockPartitions( MatrixBlock mb, boolean colwise )
	{
		MatrixBlock[] ret = null;
		int rows = mb.getNumRows();
		int cols = mb.getNumColumns();
		long nnz = mb.getNonZeros();
		boolean sparse = mb.isInSparseFormat();
		double sparsity = ((double)nnz)/(rows*cols);

		if( colwise ) //COL PARTITIONS
		{
			//allocate output partitions
			ret = new MatrixBlock[ cols ];
			for( int j=0; j<cols; j++ )
				ret[j] = new MatrixBlock(rows, 1, false);

			//cache-friendly sequential read/append
			if( !mb.isEmptyBlock(false) ) {
				if( sparse ){ //SPARSE
					Iterator<IJV> iter = mb.getSparseBlockIterator();
					while( iter.hasNext() ) {
						IJV cell = iter.next();
						ret[cell.getJ()].appendValue(cell.getI(), 0, cell.getV());
					}
				}
				else { //DENSE
					DenseBlock d = mb.getDenseBlock();
					for( int i=0; i<rows; i++ )
						for( int j=0; j<cols; j++ )
							ret[j].appendValue(i, 0, d!=null?d.get(i, j) : 0);
				}
			}
		}
		else //ROW PARTITIONS
		{
			//allocate output partitions
			ret = new MatrixBlock[ rows ];
			for( int i=0; i<rows; i++ )
				ret[i] = new MatrixBlock(1, cols, sparse, (long)(cols*sparsity));

			//cache-friendly sparse/dense row slicing
			if( !mb.isEmptyBlock(false) ) {
				for( int i=0; i<rows; i++ )
					mb.slice(i, i, 0, cols-1, ret[i]);
			}
		}

		return ret;
	}

	/**
	 * Helper method that converts SystemDS matrix variable (<code>varname</code>) into a Array2DRowRealMatrix format,
	 * which is useful in invoking Apache CommonsMath.
	 *
	 * @param mb matrix object
	 * @return matrix as a commons-math3 Array2DRowRealMatrix
	 */
	public static Array2DRowRealMatrix convertToArray2DRowRealMatrix(MatrixBlock mb) {
		double[][] data = DataConverter.convertToDoubleMatrix(mb);
		return new Array2DRowRealMatrix(data, false);
	}

	public static BlockRealMatrix convertToBlockRealMatrix(MatrixBlock mb) {
		BlockRealMatrix ret = new BlockRealMatrix(mb.getNumRows(), mb.getNumColumns());
		if( mb.getNonZeros() > 0 ) {
			if( mb.isInSparseFormat() ) {
				Iterator<IJV> iter = mb.getSparseBlockIterator();
				while( iter.hasNext() ) {
					IJV cell = iter.next();
					ret.setEntry(cell.getI(), cell.getJ(), cell.getV());
				}
			}
			else {
				double[] a = mb.getDenseBlockValues();
				for( int i=0, ix=0; i<mb.getNumRows(); i++ )
					for( int j=0; j<mb.getNumColumns(); j++, ix++ )
						ret.setEntry(i, j, a[ix]);
			}
		}
		return ret;
	}

	public static MatrixBlock convertToMatrixBlock(RealMatrix rm) {
		MatrixBlock ret = new MatrixBlock(rm.getRowDimension(),
			rm.getColumnDimension(), false).allocateDenseBlock();
		for(int i=0; i<ret.getNumRows(); i++)
			for(int j=0; j<ret.getNumColumns(); j++)
				ret.set(i, j, rm.getEntry(i, j));
		ret.examSparsity();
		return ret;
	}

	public static void copyToDoubleVector( MatrixBlock mb, double[] dest, int destPos )
	{
		if( mb.isEmptyBlock(false) )
			return; //quick path

		int rows = mb.getNumRows();
		int cols = mb.getNumColumns();

		if( mb.isInSparseFormat() ) {
			Iterator<IJV> iter = mb.getSparseBlockIterator();
			while( iter.hasNext() ) {
				IJV cell = iter.next();
				dest[destPos+cell.getI()*cols+cell.getJ()] = cell.getV();
			}
		}
		else {
			//memcopy row major representation if at least 1 non-zero
			System.arraycopy(mb.getDenseBlockValues(), 0, dest, destPos, rows*cols);
		}
	}

	/**
	 * Convenience method to print NaN & Infinity compliant with how as.scalar prints them.
	 * {@link DecimalFormat} prints NaN as \uFFFD and Infinity as \u221E
	 * http://docs.oracle.com/javase/6/docs/api/java/text/DecimalFormat.html
	 * @param df	The {@link DecimalFormat} instance, constructed with the appropriate options
	 * @param value	The double value to print
	 * @return	a string formatted with the {@link DecimalFormat} instance or "NaN" or "Infinity" or "-Infinity"
	 */
	private static String dfFormat(DecimalFormat df, double value) {
		if (Double.isNaN(value) || Double.isInfinite(value)){
			return Double.toString(value);
		} else {
			return df.format(value);
		}
	}

	/**
	 * Creates a non-grouping {@link DecimalFormat} for printing values. When {@code decimal >= 0}
	 * both the minimum and maximum fraction digits are pinned to {@code decimal}, so values are
	 * printed with exactly that many decimals; otherwise the {@link DecimalFormat} defaults apply.
	 * @param decimal number of decimal places to print, -1 for default
	 * @return a configured {@link DecimalFormat}
	 */
	private static DecimalFormat createDecimalFormat(int decimal) {
		DecimalFormat df = new DecimalFormat();
		df.setGroupingUsed(false);
		if (decimal >= 0) {
			df.setMinimumFractionDigits(decimal);
			df.setMaximumFractionDigits(decimal);
		}
		return df;
	}

	public static String toString(MatrixBlock mb) {
		return toString(mb, false, " ", "\n", mb.getNumRows(), mb.getNumColumns(), 3);
	}

	/**
	 * Returns a string representation of a matrix
	 * @param mb matrix block
	 * @param sparse if true, string will contain a table with row index, col index, value (where value != 0.0)
	 * 				 otherwise it will be a rectangular string with all values of the matrix block
	 * @param separator Separator string between each element in a row, or between the columns in sparse format
	 * @param lineseparator Separator string between each row
	 * @param rowsToPrint maximum number of rows to print, -1 for all
	 * @param colsToPrint maximum number of columns to print, -1 for all
	 * @param decimal number of decimal places to print, -1 for default
	 * @return matrix as a string
	 */
	public static String toString(MatrixBlock mb, boolean sparse, String separator, String lineseparator, int rowsToPrint, int colsToPrint, int decimal){
		StringBuffer sb = new StringBuffer();

		// Setup number of rows and columns to print
		int rlen = mb.getNumRows();
		int clen = mb.getNumColumns();
		int rowLength = rlen;
		int colLength = clen;
		if (rowsToPrint >= 0)
			rowLength = rowsToPrint < rlen ? rowsToPrint : rlen;
		if (colsToPrint >= 0)
			colLength = colsToPrint < clen ? colsToPrint : clen;

		DecimalFormat df = createDecimalFormat(decimal);

		if (sparse){ // Sparse Print Format
			if (mb.isInSparseFormat()){	// Block is in sparse format
				Iterator<IJV> sbi = mb.getSparseBlockIterator();
				while (sbi.hasNext()){
					IJV ijv = sbi.next();
					int row = ijv.getI();
					int col = ijv.getJ();
					double value = ijv.getV();
					if (row < rowLength && col < colLength) {
						// Print (row+1) and (col+1) since for a DML user, everything is 1-indexed
						sb.append(row+1).append(separator).append(col+1).append(separator);
						sb.append(dfFormat(df, value)).append(lineseparator);
					}
				}
			} else {	// Block is in dense format
				for (int i=0; i<rowLength; i++){
					for (int j=0; j<colLength; j++){
						double value = mb.get(i, j);
						if (value != 0.0){
							sb.append(i+1).append(separator).append(j+1).append(separator);
							sb.append(dfFormat(df, value)).append(lineseparator);
						}
					}
				}
			}
		}
		else {	// Dense Print Format
			for (int i=0; i<rowLength; i++){
				for (int j=0; j<colLength-1; j++){
					Double value = mb.get(i, j);
					if (value.equals(-0.0d))
						value = 0.0;
					sb.append(dfFormat(df, value));
					sb.append(separator);
				}
				Double value = mb.get(i, colLength-1);
				if (value.equals(-0.0d))
					value = 0.0;
				sb.append(dfFormat(df, value));	// Do not put separator after last element
				sb.append(lineseparator);
			}
		}

		return sb.toString();
	}

	public static String toString(TensorBlock tb) {
		return toString(tb, false, " ", "\n", "[", "]", tb.getDim(0), tb.getDim(1), 3);
	}

	/**
	 * Returns a string representation of a tensor
	 * @param tb tensor block
	 * @param sparse if true, string will contain a table with row index, col index, value (where value != 0.0)
	 * 				 otherwise it will be a rectangular string with all values of the tensor block
	 * @param separator Separator string between each element in a row, or between the columns in sparse format
	 * @param lineseparator Separator string between each row
	 * @param leftBorder Characters placed at the start of a new dimension level
	 * @param rightBorder Characters placed at the end of a new dimension level
	 * @param rowsToPrint maximum number of rows to print, -1 for all
	 * @param colsToPrint maximum number of columns to print, -1 for all
	 * @param decimal number of decimal places to print, -1 for default
	 * @return tensor as a string
	 */
	public static String toString(TensorBlock tb, boolean sparse, String separator, String lineseparator,
		String leftBorder, String rightBorder, int rowsToPrint, int colsToPrint, int decimal){
		StringBuilder sb = new StringBuilder();

		// Setup number of rows and columns to print
		int rlen = tb.getDim(0);
		int clen = tb.getDim(1);
		int rowLength = rlen;
		int colLength = clen;
		if (rowsToPrint >= 0)
			rowLength = Math.min(rowsToPrint, rlen);
		if (colsToPrint >= 0)
			colLength = Math.min(colsToPrint, clen);

		DecimalFormat df = createDecimalFormat(decimal);

		if (sparse){ // Sparse Print Format
			// TODO use sparse iterator for sparse block
			int[] ix = new int[tb.getNumDims()];
			for (int i = 0; i < tb.getLength(); i++) {
				String str = tb.get(ix).toString();
				if (str != null && !str.isEmpty() && Double.parseDouble(str) != 0) {
					for (int item : ix) {
						sb.append(item).append(separator);
					}
					concatenateTensorValue(tb, sb, df, ix);
					sb.append(lineseparator);
				}
				TensorBlock.getNextIndexes(tb.getDims(), ix);
				if (ix[0] >= rowLength) {
					break;
				}
			}
		}
		else {	// Dense Print Format
			int[] ix = new int[tb.getNumDims()];
			sb.append(StringUtils.repeat(leftBorder, ix.length));
			for (int i = 0; i < tb.getLength(); i++) {
				concatenateTensorValue(tb, sb, df, ix);
				int j = ix.length - 1;
				ix[j]++;
				//calculating next index
				int borderCount = 0;
				while (ix[j] == tb.getDim(j) || ix[1] >= colLength) {
					// we either reached the dimension limit or the colLength if j == 1
					// so we add border (because of the completely iterated dimension) and increment the next
					// dimension while setting the current one to 0
					if (ix[j] != tb.getDim(j))
						sb.append("...");
					sb.append(rightBorder);
					borderCount++;
					ix[j] = 0;
					j--;
					if (j < 0) {
						break;
					}
					ix[j]++;
				}
				if (ix[0] >= rowLength) {
					// If we have a limit on rows end here
					sb.append("...").append(rightBorder).append(lineseparator);
					break;
				}
				if (j < 0) {
					// we are at the end, no offset
					sb.append(lineseparator);
					break;
				}
				if (borderCount == 0) {
					sb.append(separator);
				} else {
					// Offset so dimensions are aligned
					sb.append(lineseparator);
					sb.append(StringUtils.repeat(" ", (ix.length - borderCount) * leftBorder.length()));
					sb.append(StringUtils.repeat(leftBorder, borderCount));
				}
			}
		}
		return sb.toString();
	}

	/**
	 * Concatenates a single tensor value to the `StringBuilder` by converting it to the correct format.
	 *
	 * @param tb the BasicTensor
	 * @param sb the StringBuilder to use
	 * @param df DecimalFormat with the correct settings for double or float values
	 * @param ix the index of the TensorBlock value
	 */
	private static void concatenateTensorValue(TensorBlock tb, StringBuilder sb, DecimalFormat df, int[] ix) {
		switch (tb.isBasic() ? tb.getValueType() : tb.getSchema()[ix[1]]) {
			case FP32:
				Float valuef = (Float) tb.get(ix);
				if (valuef.equals(-0.0f))
					valuef = 0.0f;
				sb.append(dfFormat(df, valuef));
				break;
			case FP64:
				Double value = (Double) tb.get(ix);
				if (value.equals(-0.0d))
					value = 0.0;
				sb.append(dfFormat(df, value));
				break;
			case UINT8:
			case UINT4:
			case INT32:
			case INT64:
				sb.append(tb.get(ix));
				break;
			case BOOLEAN:
				sb.append(((Boolean) tb.get(ix)).toString().toUpperCase());
				break;
			default:
				sb.append("\"").append(tb.get(ix)).append("\"");
				break;
		}
	}

	public static String toString(FrameBlock fb) {
		return toString(fb, false, " ", "\n", fb.getNumRows(), fb.getNumColumns(), 3);
	}

	public static String toString(FrameBlock fb, boolean sparse, String separator, String lineseparator, int rowsToPrint, int colsToPrint, int decimal)
	{
		StringBuffer sb = new StringBuffer();

		// Setup number of rows and columns to print
		int rlen = fb.getNumRows();
		int clen = fb.getNumColumns();
		int rowLength = rlen;
		int colLength = clen;
		if (rowsToPrint >= 0)
			rowLength = rowsToPrint < rlen ? rowsToPrint : rlen;
		if (colsToPrint >= 0)
			colLength = colsToPrint < clen ? colsToPrint : clen;

		//print frame header
		sb.append("# FRAME: ");
		sb.append("nrow = " + fb.getNumRows() + ", ");
		sb.append("ncol = " + fb.getNumColumns() + lineseparator);

		//print column names
		sb.append("#"); sb.append(separator);
		for( int j=0; j<colLength; j++ ) {
			sb.append(fb.getColumnNames()[j]);
			if( j != colLength-1 )
				sb.append(separator);
		}
		sb.append(lineseparator);

		//print schema
		sb.append("#"); sb.append(separator);
		for( int j=0; j<colLength; j++ ) {
			sb.append(fb.getSchema()[j]);
			if( j != colLength-1 )
				sb.append(separator);
		}
		sb.append(lineseparator);

		//print data
		DecimalFormat df = createDecimalFormat(decimal);

		Iterator<Object[]> iter = IteratorFactory.getObjectRowIterator(fb, 0, rowLength);
		while( iter.hasNext() ) {
			Object[] row = iter.next();
			for( int j=0; j<colLength; j++ ) {
				if( row[j]==null )
					sb.append(String.valueOf(row[j]));
				else if( fb.getSchema()[j] == ValueType.FP64 )
					sb.append(dfFormat(df, (Double)row[j]));
				else if( fb.getSchema()[j] == ValueType.BOOLEAN )
					sb.append(new BooleanObject((Boolean)row[j])
						.getLanguageSpecificStringValue());
				else
					sb.append(row[j]);
				if( j != colLength-1 )
					sb.append(separator);
			}
			sb.append(lineseparator);
		}

		return sb.toString();
	}

	public static String toString(ListObject list,int rows, int cols, boolean sparse, String separator, String lineSeparator, int rowsToPrint, int colsToPrint, int decimal)
	{
		StringBuilder sb = new StringBuilder();
		sb.append("List containing:\n");
		sb.append("[");
		for(Data x : list.getData()){
			if( x instanceof MatrixObject) {
				sb.append("\nMatrix:\n");
				MatrixObject dat = (MatrixObject) x;
				MatrixBlock matrix = dat.acquireRead();
				sb.append(DataConverter.toString(matrix, sparse, separator, lineSeparator, rows, cols, decimal));
				dat.release();
			}
			else if( x instanceof TensorObject ) {
				sb.append("\n");
				TensorObject dat = (TensorObject) x;
				TensorBlock tensor = dat.acquireRead();
				sb.append(DataConverter.toString(tensor, sparse, separator,
					lineSeparator, "[", "]", rows, cols, decimal));
				dat.release();
			}
			else if( x instanceof FrameObject ) {
				sb.append("\n");
				FrameObject dat = (FrameObject) x;
				FrameBlock frame = dat.acquireRead();
				sb.append(DataConverter.toString(frame, sparse, separator, lineSeparator, rows, cols, decimal));
				dat.release();
			}
			else if (x instanceof ListObject){
				ListObject dat = (ListObject) x;
				sb.append(DataConverter.toString(dat, cols, rows,sparse, separator, lineSeparator, rows, cols, decimal));
			}else{
				sb.append(x.toString());
			}
			sb.append(", ");
		}
		sb.delete(sb.length() -2, sb.length());
		sb.append("]");
		return sb.toString();
	}

	public static int[] getTensorDimensions(ExecutionContext ec, CPOperand dims) {
		int[] tDims;
		switch (dims.getDataType()) {
			case SCALAR: {
				// Dimensions given as string
				if (dims.getValueType() != Types.ValueType.STRING) {
					throw new DMLRuntimeException("Dimensions have to be passed as list, string, matrix or tensor.");
				}
				String dimensionString = ec.getScalarInput(dims.getName(), Types.ValueType.STRING, dims.isLiteral())
					.getStringValue();
				StringTokenizer dimensions = new StringTokenizer(dimensionString, DELIM);
				tDims = new int[dimensions.countTokens()];
				Arrays.setAll(tDims, (i) -> Integer.parseInt(dimensions.nextToken()));
			}
			break;
			case MATRIX: {
				// Dimensions given as vector
				MatrixBlock in = ec.getMatrixInput(dims.getName());
				boolean colVec = false;
				if (in.getNumRows() == 1) {
					colVec = true;
				} else if (!(in.getNumColumns() == 1)) {
					throw new DMLRuntimeException("Dimensions matrix has to be a vector.");
				}
				tDims = new int[(int) in.getLength()];
				for (int i = 0; i < in.getLength(); i++) {
					tDims[i] = UtilFunctions.toInt(in.get(colVec ? 0 : i, colVec ? i : 0));
				}
				ec.releaseMatrixInput(dims.getName());
			}
			break;
			case TENSOR: {
				// Dimensions given as vector
				TensorBlock in = ec.getTensorInput(dims.getName());
				boolean colVec = false;
				if (!in.isVector()) {
					throw new DMLRuntimeException("Dimensions tensor has to be a vector.");
				} else if (in.getNumRows() == 1) {
					colVec = true;
				}
				tDims = new int[(int) in.getLength()];
				for (int i = 0; i < in.getLength(); i++) {
					tDims[i] = UtilFunctions.toInt(in.get(new int[]{colVec ? 0 : i, colVec ? i : 0}));
				}
				ec.releaseTensorInput(dims.getName());
			}
			break;
			case LIST: {
				// Dimensions given as List
				ListObject list = ec.getListObject(dims.getName());
				tDims = new int[list.getLength()];
				List<Data> dimsData = list.getData();
				for (int i = 0; i < tDims.length; i++) {
					if (dimsData.get(i) instanceof ScalarObject) {
						// TODO warning if double value is cast to long?
						tDims[i] = (int) ((ScalarObject) dimsData.get(i)).getLongValue();
					} else {
						throw new DMLRuntimeException("Dims parameter for does not support lists with non scalar values.");
					}
				}
			}
			break;
			default:
				throw new DMLRuntimeException("Dimensions have to be passed as list, string, matrix or tensor.");
		}
		return tDims;
	}

	public static double[] toDouble(float[] data) {
		double[] ret = new double[data.length];
		for(int i=0; i<data.length; i++)
			ret[i] = data[i];
		return ret;
	}

	public static double[] toDouble(long[] data) {
		double[] ret = new double[data.length];
		for(int i=0; i<data.length; i++)
			ret[i] = data[i];
		return ret;
	}

	public static double[] toDouble(int[] data) {
		double[] ret = new double[data.length];
		for(int i=0; i<data.length; i++)
			ret[i] = data[i];
		return ret;
	}

	public static double[] toDouble(BitSet data, int len) {
		double[] ret = new double[len];
		for(int i=0; i<len; i++)
			ret[i] = data.get(i) ? 1 : 0;
		return ret;
	}

	public static double[] toDouble(String[] data) {
		double[] ret = new double[data.length];
		for(int i=0; i<data.length; i++)
			ret[i] = data[i] == null || data[i].isEmpty() ? 0 : Double.parseDouble(data[i]);
		return ret;
	}

	public static float[] toFloat(double[] data) {
		float[] ret = new float[data.length];
		for( int i=0; i<data.length; i++ )
			ret[i] = (float)data[i];
		return ret;
	}

	public static int[] toInt(double[] data) {
		int[] ret = new int[data.length];
		for( int i=0; i<data.length; i++ )
			ret[i] = UtilFunctions.toInt(data[i]);
		return ret;
	}

	public static long[] toLong(double[] data) {
		long[] ret = new long[data.length];
		for( int i=0; i<data.length; i++ )
			ret[i] = UtilFunctions.toLong(data[i]);
		return ret;
	}

	public static BitSet toBitSet(double[] data) {
		BitSet ret = new BitSet(data.length);
		for( int i=0; i<data.length; i++ )
			ret.set(i, data[i] != 0);
		return ret;
	}

	public static String[] toString(double[] data) {
		String[] ret = new String[data.length];
		for( int i=0; i<data.length; i++ )
			ret[i] = String.valueOf(data[i]);
		return ret;
	}
}
