DataFrameTransformation

Value Members

1. final def !=(arg0: Any): Boolean

   

   Definition Classes

   Any

2. final def ##(): Int

   

   Definition Classes

   Any

3. final def ==(arg0: Any): Boolean

   

   Definition Classes

   Any

4. final def asInstanceOf[T0]: T0

   

   Definition Classes

   Any

5. def castToLong: DataFrame

   

   Cast all integer columns in a DataFrame to long.

   Cast all integer columns in a DataFrame to long.

   returns

   a DataFrame with all integer values cast to long

6. def decodeHexStringToBinary(hexColumns: Set[String]): DataFrame

   

7. val df: DataFrame

   

8. def encodeBinaryToHexString: DataFrame

   

9. def encodeIdColumns(idColumns: String*): Seq[Column]

   

10. def getClass(): Class[_ <: AnyVal]

    

    Definition Classes

    AnyVal → Any

11. final def isInstanceOf[T0]: Boolean

    

    Definition Classes

    Any

12. def prefixColumns(prefix: String): DataFrame

    

13. def removePrefix(prefix: String): DataFrame

    

14. def safeAddColumn(name: String, col: Column): DataFrame

    

15. def safeAddColumns(columns: (String, Column)*): DataFrame

    

16. def safeDropColumns(names: String*): DataFrame

    

17. def safeJoin(other: DataFrame, joinCols: Seq[(String, String)], joinType: String): DataFrame

    

18. def safeRenameColumns(renamings: Map[String, String]): DataFrame

    

19. def safeRenameColumns(renamings: (String, String)*): DataFrame

    

20. def safeReplaceColumn(name: String, newColumn: Column): DataFrame

    

21. def toString(): String

    

    Definition Classes

    Any

22. def transformColumns(cols: String*)(f: (Column) ⇒ Column): DataFrame

    

23. def withCypherCompatibleTypes: DataFrame

    

    Normalises the DataFrame by lifting numeric fields to Long and similar ops.

24. def withHashColumn(columns: Seq[Column], hashColumn: String): DataFrame

    

    Adds a new column hashColumn containing the hash value of the given input columns.

    Adds a new column hashColumn containing the hash value of the given input columns.

    The hash is generated using org.apache.spark.sql.catalyst.expressions.Murmur3Hash based on the given column sequence. To decrease collision probability, we:

    1) generate a first hash for the given column sequence 2) shift the hash into the upper bits of a 64 bit long 3) generate a second hash using the reversed input column sequence 4) store the hash in the lower 32 bits of the final id

    columns

    input columns for the hash function

    hashColumn

    column storing the result of the hash function

    returns

    DataFrame with an additional column that contains the hash ID

25. def withSerializedIdColumn(columns: Seq[Column], serializedColumn: String): DataFrame

    

    Adds a new column serializedColumn containing the serialized values of the given input columns.

    Adds a new column serializedColumn containing the serialized values of the given input columns.

    columns

    input columns for the serialization function

    serializedColumn

    column storing the result of the serialization function

    returns

    DataFrame with an additional column that contains the serialized ID