eu.dnetlib.dhp.oa.dedup

Class SparkUpdateEntity

java.lang.Object

eu.dnetlib.dhp.oa.dedup.SparkUpdateEntity

All Implemented Interfaces:

Serializable

public class SparkUpdateEntity
extends Object

See Also:

Serialized Form

Field Summary

Fields

Modifier and Type	Field and Description
protected static int	NUM_CONNECTIONS
protected static int	NUM_PARTITIONS
protected static com.fasterxml.jackson.databind.ObjectMapper	OBJECT_MAPPER
eu.dnetlib.dhp.application.ArgumentApplicationParser	parser
protected static String	SP_SEPARATOR
org.apache.spark.sql.SparkSession	spark
protected static String	TYPE_VALUE_SEPARATOR

Constructor Summary

Constructors

Constructor and Description
SparkUpdateEntity(eu.dnetlib.dhp.application.ArgumentApplicationParser parser, org.apache.spark.sql.SparkSession spark)

Method Summary

Modifier and Type	Method and Description
List<eu.dnetlib.pace.config.DedupConfig>	getConfigurations(eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService isLookUpService, String orchestrator)
protected static org.apache.spark.sql.SparkSession	getSparkSession(org.apache.spark.SparkConf conf)
protected boolean	isOpenorgs(eu.dnetlib.dhp.schema.oaf.Relation rel)
protected boolean	isOpenorgsDedupRel(eu.dnetlib.dhp.schema.oaf.Relation rel)
static void	main(String[] args)
boolean	mergeRelExists(String basePath, String entity)
protected static Boolean	parseECField(eu.dnetlib.dhp.schema.oaf.Field<String> field)
protected static org.apache.spark.api.java.function.MapFunction<String,eu.dnetlib.dhp.schema.oaf.Relation>	patchRelFn()
protected static void	removeOutputDir(org.apache.spark.sql.SparkSession spark, String path)
void	run(eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService isLookUpService)
protected static <T> void	save(org.apache.spark.sql.Dataset<T> dataset, String outPath, org.apache.spark.sql.SaveMode mode)
protected static <T> void	saveParquet(org.apache.spark.sql.Dataset<T> dataset, String outPath, org.apache.spark.sql.SaveMode mode)
protected static String	structuredPropertyListToString(List<eu.dnetlib.dhp.schema.oaf.StructuredProperty> list)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

NUM_PARTITIONS

protected static final int NUM_PARTITIONS

See Also:

Constant Field Values

NUM_CONNECTIONS

protected static final int NUM_CONNECTIONS

See Also:

Constant Field Values

TYPE_VALUE_SEPARATOR

protected static final String TYPE_VALUE_SEPARATOR

See Also:

Constant Field Values

SP_SEPARATOR

protected static final String SP_SEPARATOR

See Also:

Constant Field Values

OBJECT_MAPPER

protected static final com.fasterxml.jackson.databind.ObjectMapper OBJECT_MAPPER

parser

public final eu.dnetlib.dhp.application.ArgumentApplicationParser parser

spark

public final org.apache.spark.sql.SparkSession spark

Constructor Detail

SparkUpdateEntity

public SparkUpdateEntity(eu.dnetlib.dhp.application.ArgumentApplicationParser parser,
                         org.apache.spark.sql.SparkSession spark)

Method Detail

main

public static void main(String[] args)
                 throws Exception

Throws:

Exception

run

public void run(eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService isLookUpService)
         throws IOException

Throws:

IOException

mergeRelExists

public boolean mergeRelExists(String basePath,
                              String entity)
                       throws IOException

Throws:

IOException

getConfigurations

public List<eu.dnetlib.pace.config.DedupConfig> getConfigurations(eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService isLookUpService,
                                                                  String orchestrator)
                                                           throws eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException,
                                                                  org.dom4j.DocumentException,
                                                                  IOException,
                                                                  SAXException

Throws:

eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException

org.dom4j.DocumentException

IOException

SAXException

getSparkSession

protected static org.apache.spark.sql.SparkSession getSparkSession(org.apache.spark.SparkConf conf)

save

protected static <T> void save(org.apache.spark.sql.Dataset<T> dataset,
                               String outPath,
                               org.apache.spark.sql.SaveMode mode)

saveParquet

protected static <T> void saveParquet(org.apache.spark.sql.Dataset<T> dataset,
                                      String outPath,
                                      org.apache.spark.sql.SaveMode mode)

removeOutputDir

protected static void removeOutputDir(org.apache.spark.sql.SparkSession spark,
                                      String path)

structuredPropertyListToString

protected static String structuredPropertyListToString(List<eu.dnetlib.dhp.schema.oaf.StructuredProperty> list)

patchRelFn

protected static org.apache.spark.api.java.function.MapFunction<String,eu.dnetlib.dhp.schema.oaf.Relation> patchRelFn()

isOpenorgs

protected boolean isOpenorgs(eu.dnetlib.dhp.schema.oaf.Relation rel)

isOpenorgsDedupRel

protected boolean isOpenorgsDedupRel(eu.dnetlib.dhp.schema.oaf.Relation rel)

parseECField

protected static Boolean parseECField(eu.dnetlib.dhp.schema.oaf.Field<String> field)