Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: marklogic/marklogic-contentpump
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 10.0.8
Choose a base ref
...
head repository: marklogic/marklogic-contentpump
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: 10.0.8.2
Choose a head ref
  • 4 commits
  • 6 files changed
  • 2 contributors

Commits on Dec 24, 2021

  1. Bug:57185 - Upgrade log4j to log4j2

    Vanessa Zhang authored and Vanessa Zhang committed Dec 24, 2021
    Copy the full SHA
    fe246e6 View commit details

  2. Bug:57185 - Add back jackon-annotations accidentally removed

    Vanessa Zhang authored and Vanessa Zhang committed Dec 24, 2021
    Copy the full SHA
    7935e00 View commit details

  3. Bug:57185 - Add back avro accidentally removed

    Vanessa Zhang authored and Vanessa Zhang committed Dec 24, 2021
    Copy the full SHA
    22cb023 View commit details

Commits on Dec 25, 2021

  1. Updating version to be mlcp: 10.0.8.2

    @itsshivaverma
    itsshivaverma committed Dec 25, 2021
    Copy the full SHA
    2096240 View commit details
Showing with 149 additions and 47 deletions.
  1. +81 −13 pom.xml
  2. +10 −7 src/assemble/bindist-mapr.xml
  3. +10 −7 src/assemble/bindist.xml
  4. +0 −15 src/conf/log4j.properties
  5. +43 −0 src/conf/log4j2.xml
  6. +5 −5 src/test/java/com/marklogic/contentpump/TestImportBigData.java
94 changes: 81 additions & 13 deletions pom.xml
View file
Original file line number Diff line number Diff line change
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.marklogic</groupId>
<artifactId>mlcp</artifactId>
<version>10.0.8</version>
<version>10.0.8.2</version>
<name>MarkLogic Content Pump</name>
<description>MarkLogic Content Pump</description>
<url>https://github.com/marklogic/marklogic-contentpump</url>
@@ -91,6 +91,13 @@
<additionalClasspathElements>
<additionalClasspathElement>src/conf</additionalClasspathElement>
</additionalClasspathElements>
<!-- Add dependency exclusions to avoid log4j 1.X being called and slf4j multiple
binding issues -->
<classpathDependencyExcludes>
<classpathDependencyExclude>log4j:log4j</classpathDependencyExclude>
<classpathDependencyExclude>org.slf4j:slf4j-log4j12</classpathDependencyExclude>
<classpathDependencyExclude>ch.qos.logback:logback-classic</classpathDependencyExclude>
</classpathDependencyExcludes>
<skipTests>${skipTests}</skipTests>
<argline>-DCONTENTPUMP_HOME=${contentpumpHome} -DOUTPUT_PATH=${testOutputPath} -DTEST_DB_PORT=${testDbPort} -DCOPY_DB_PORT=${copyDbPort} -DTEST_DB=${testDb} -DCOPY_DB=${copyDb}</argline>
<excludes>
@@ -467,14 +474,19 @@
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<artifactId>jackson-annotations</artifactId>
<version>2.11.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.11.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.11.0</version>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
@@ -495,11 +507,6 @@
<artifactId>commons-modeler</artifactId>
<version>2.0.1</version>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro-tools</artifactId>
<version>1.8.1</version>
</dependency>
<dependency>
<groupId>info.bliki.wiki</groupId>
<artifactId>bliki-core</artifactId>
@@ -535,6 +542,36 @@
<artifactId>httpclient</artifactId>
<version>4.2.3</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
<version>2.17.0</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.17.0</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.17.0</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-jcl</artifactId>
<version>2.17.0</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<version>2.17.0</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.32</version>
</dependency>
<!-- System dependencies-->
<dependency>
<groupId>org.apache.commons</groupId>
@@ -647,14 +684,19 @@
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<artifactId>jackson-annotations</artifactId>
<version>2.11.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.11.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.11.0</version>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
@@ -690,11 +732,6 @@
<artifactId>commons-modeler</artifactId>
<version>2.0.1</version>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro-tools</artifactId>
<version>1.8.1</version>
</dependency>
<dependency>
<groupId>info.bliki.wiki</groupId>
<artifactId>bliki-core</artifactId>
@@ -715,6 +752,37 @@
<artifactId>httpclient</artifactId>
<version>4.2.3</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
<version>2.17.0</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.17.0</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.17.0</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-jcl</artifactId>
<version>2.17.0</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
<version>2.17.0</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.32</version>
</dependency>

<!-- System dependencies-->
<dependency>
<groupId>org.apache.commons</groupId>
17 changes: 10 additions & 7 deletions src/assemble/bindist-mapr.xml
View file
Original file line number Diff line number Diff line change
@@ -22,7 +22,7 @@
<outputDirectory>${file.separator}conf</outputDirectory>
<fileMode>0755</fileMode>
<includes>
<include>log4j.properties</include>
<include>log4j2.xml</include>
<include>xcc.logging.properties</include>
</includes>
</fileSet>
@@ -81,8 +81,8 @@
<include>com.sun.jersey:jersey-core:jar:*</include>
<include>com.sun.jersey:jersey-client:jar:*</include>
<include>commons-collections:commons-collections:jar:*</include>
<include>com.marklogic:*:*:*</include>
<include>org.apache.avro:avro:jar:*</include>
<include>com.marklogic:*:*:*</include>
<include>org.apache.avro:avro:jar:*</include>
<include>commons-cli:commons-cli:jar:*</include>
<include>commons-codec:commons-codec:jar:*</include>
<include>commons-configuration:commons-configuration:jar:*</include>
@@ -101,13 +101,16 @@
<include>org.apache.hadoop:hadoop-yarn-common:jar:*mapr*</include>
<include>org.apache.hadoop:hadoop-yarn-client:jar:*mapr*</include>
<include>org.apache.hadoop:hadoop-mapreduce-client-contrib:jar:*mapr*</include>
<include>com.fasterxml.jackson.core:jackson-annotations:jar:*</include>
<include>com.fasterxml.jackson.core:jackson-core:jar:*</include>
<include>com.fasterxml.jackson.core:jackson-databind:jar:*</include>
<include>com.fasterxml.jackson.core:jackson-annotations:jar:*</include>
<include>log4j:log4j:jar:*</include>
<include>com.google.protobuf:protobuf-java:jar:*</include>
<include>org.apache.logging.log4j:log4j-1.2-api:jar:*</include>
<include>org.apache.logging.log4j:log4j-api:jar:*</include>
<include>org.apache.logging.log4j:log4j-core:jar:*</include>
<include>org.apache.logging.log4j:log4j-jcl:jar:*</include>
<include>org.apache.logging.log4j:log4j-slf4j-impl:jar:*</include>
<include>org.slf4j:slf4j-api:jar:*</include>
<include>org.slf4j:slf4j-log4j12:jar:*</include>
<include>com.google.protobuf:protobuf-java:jar:*</include>
<include>xerces:xercesImpl:jar:*</include>
<include>xpp3:xpp3:jar:*</include>
<include>com.github.jsonld-java:jsonld-java:jar:*</include>
17 changes: 10 additions & 7 deletions src/assemble/bindist.xml
View file
Original file line number Diff line number Diff line change
@@ -22,7 +22,7 @@
<outputDirectory>${file.separator}conf</outputDirectory>
<fileMode>0755</fileMode>
<includes>
<include>log4j.properties</include>
<include>log4j2.xml</include>
<include>xcc.logging.properties</include>
</includes>
</fileSet>
@@ -81,8 +81,8 @@
<include>com.sun.jersey:jersey-core:jar:*</include>
<include>com.sun.jersey:jersey-client:jar:*</include>
<include>commons-collections:commons-collections:jar:*</include>
<include>com.marklogic:*:*:*</include>
<include>org.apache.avro:avro:jar:*</include>
<include>com.marklogic:*:*:*</include>
<include>org.apache.avro:avro:jar:*</include>
<include>commons-cli:commons-cli:jar:*</include>
<include>commons-codec:commons-codec:jar:*</include>
<include>commons-configuration:commons-configuration:jar:*</include>
@@ -100,13 +100,16 @@
<include>org.apache.hadoop:hadoop-yarn-api:jar:*</include>
<include>org.apache.hadoop:hadoop-yarn-client:jar:*</include>
<include>org.apache.hadoop:hadoop-yarn-common:jar:*</include>
<include>com.fasterxml.jackson.core:jackson-annotations:jar:*</include>
<include>com.fasterxml.jackson.core:jackson-core:jar:*</include>
<include>com.fasterxml.jackson.core:jackson-databind:jar:*</include>
<include>com.fasterxml.jackson.core:jackson-annotations:jar:*</include>
<include>log4j:log4j:jar:*</include>
<include>com.google.protobuf:protobuf-java:jar:*</include>
<include>org.apache.logging.log4j:log4j-1.2-api:jar:*</include>
<include>org.apache.logging.log4j:log4j-api:jar:*</include>
<include>org.apache.logging.log4j:log4j-core:jar:*</include>
<include>org.apache.logging.log4j:log4j-jcl:jar:*</include>
<include>org.apache.logging.log4j:log4j-slf4j-impl:jar:*</include>
<include>org.slf4j:slf4j-api:jar:*</include>
<include>org.slf4j:slf4j-log4j12:jar:*</include>
<include>com.google.protobuf:protobuf-java:jar:*</include>
<include>xerces:xercesImpl:jar:*</include>
<include>xpp3:xpp3:jar:*</include>
<include>com.github.jsonld-java:jsonld-java:jar:*</include>
15 changes: 0 additions & 15 deletions src/conf/log4j.properties
View file

This file was deleted.

43 changes: 43 additions & 0 deletions src/conf/log4j2.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
<!-- This is the MLCP logging properties file for log4j2 configurations. To
enable debug for mapreduce and contentpump, please uncomment the two
logger blocks.
-->
<Configuration status="WARN">
<Appenders>
<!-- Console appender configuration -->
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n"/>
</Console>
</Appenders>
<Loggers>
<!-- Root logger configuration -->
<Root level="INFO">
<AppenderRef ref="Console"/>
</Root>
<!-- To enable debug for mapreduce -->
<!--
<Logger name="com.marklogic.mapreduce" level="DEBUG" additivity="false">
<AppenderRef ref="Console"/>
</Logger>
-->
<!-- To enable debug for contentpump -->
<!--
<Logger name="com.marklogic.contentpump" level="DEBUG" additivity="false">
<AppenderRef ref="Console"/>
</Logger>
-->
<!-- To enable debug for tree -->
<!--
<Logger name="com.marklogic.tree" level="TRACE" additivity="false">
<AppenderRef ref="Console"/>
</Logger>
-->
<!-- To supress not native warn on Mac and Solaris -->
<Logger name="org.apache.hadoop.util.NativeCodeLoader" level="ERROR" additivity="false">
<AppenderRef ref="Console"/>
</Logger>
<Logger name="org.apache.hadoop.ipc.Client" level="ERROR" additivity="false">
<AppenderRef ref="Console"/>
</Logger>
</Loggers>
</Configuration>
10 changes: 5 additions & 5 deletions src/test/java/com/marklogic/contentpump/TestImportBigData.java
View file
Original file line number Diff line number Diff line change
@@ -8,6 +8,7 @@
import java.io.File;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

import org.apache.commons.io.FileUtils;
@@ -49,7 +50,6 @@ public void tearDown() {
public void testBug35949() throws Exception{
Callable<Void> callGetSplit = new Callable<Void>() {
public Void call() throws Exception{
SimpleTimeLimiter stl = new SimpleTimeLimiter();
Configuration conf = new Configuration();
conf.set("mapreduce.input.fileinputformat.inputdir",
Constants.TEST_PATH.toUri() + "/35949");
@@ -65,14 +65,14 @@ public Void call() throws Exception{
String pathWithoutSchema = Path.getPathWithoutSchemeAndAuthority(Constants.TEST_PATH).toString();
Utils.unzip(pathWithoutSchema + "/35949.zip", pathWithoutSchema);

SimpleTimeLimiter stl = new SimpleTimeLimiter();
SimpleTimeLimiter stl = SimpleTimeLimiter.create(Executors.newCachedThreadPool());

String system = System.getProperty("os.name");
if (!system.contains("win")) {
Stopwatch sw = new Stopwatch().start();
stl.callWithTimeout(callGetSplit, 3L, TimeUnit.MINUTES, true);
Stopwatch sw = Stopwatch.createStarted();
stl.callWithTimeout(callGetSplit, 3L, TimeUnit.MINUTES);
sw.stop();
long time = sw.elapsedTime(TimeUnit.SECONDS);
long time = sw.elapsed(TimeUnit.SECONDS);
System.out.println("Total time of listStatus: " + time + " seconds.");
assertTrue(time < 20L);
}