I need to insert data with more than 50 million lines that is in s3 in a hbase table. I am using AWS EMR to use cluster with hadoop services like hbase. I've already managed to put the s3 data in the hdfs and I need to pass this data to the hbase table. The mapreduce ImportTsv was not efficient to carry a large data load, so I found that I have to use bulk load. I'm using this tutorial for that: https://www.nodalpoint.com/bulk-load-data-to-hbase-in-oracle-big-data-appliance/
The 'org.apache.hadoop.hbase.mapreduce.ImportTsv' job that is used to convert the HDFS files into a storefile completes successfully, even though I get these errors at runtime:
log4j:ERROR setFile(null,true) call failed.
java.io.FileNotFoundException: /var/log/hbase/hbase.log (Permission denied)
at java.io.FileOutputStream.open0(Native Method)
at java.io.FileOutputStream.open(FileOutputStream.java:270)
at java.io.FileOutputStream.<init>(FileOutputStream.java:213)
at java.io.FileOutputStream.<init>(FileOutputStream.java:133)
at org.apache.log4j.FileAppender.setFile(FileAppender.java:294)
at org.apache.log4j.FileAppender.activateOptions(FileAppender.java:165)
at org.apache.log4j.DailyRollingFileAppender.activateOptions(DailyRollin gFileAppender.java:223)
at org.apache.log4j.config.PropertySetter.activate(PropertySetter.java:3 07)
at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.j ava:172)
at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.j ava:104)
at org.apache.log4j.PropertyConfigurator.parseAppender(PropertyConfigura tor.java:842)
at org.apache.log4j.PropertyConfigurator.parseCategory(PropertyConfigura tor.java:768)
at org.apache.log4j.PropertyConfigurator.configureRootCategory(PropertyC onfigurator.java:648)
at org.apache.log4j.PropertyConfigurator.doConfigure(PropertyConfigurato r.java:514)
at org.apache.log4j.PropertyConfigurator.doConfigure(PropertyConfigurato r.java:580)
at org.apache.log4j.helpers.OptionConverter.selectAndConfigure(OptionCon verter.java:526)
at org.apache.log4j.LogManager.<clinit>(LogManager.java:127)
at org.apache.log4j.Logger.getLogger(Logger.java:104)
at org.apache.commons.logging.impl.Log4JLogger.getLogger(Log4JLogger.jav a:262)
at org.apache.commons.logging.impl.Log4JLogger.<init>(Log4JLogger.java:1 08)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstruct orAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingC onstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.apache.commons.logging.impl.LogFactoryImpl.createLogFromClass(Log FactoryImpl.java:1025)
at org.apache.commons.logging.impl.LogFactoryImpl.discoverLogImplementat ion(LogFactoryImpl.java:844)
at org.apache.commons.logging.impl.LogFactoryImpl.newInstance(LogFactory Impl.java:541)
at org.apache.commons.logging.impl.LogFactoryImpl.getInstance(LogFactory Impl.java:292)
at org.apache.commons.logging.impl.LogFactoryImpl.getInstance(LogFactory Impl.java:269)
at org.apache.commons.logging.LogFactory.getLog(LogFactory.java:655)
at org.apache.hadoop.hbase.mapreduce.ImportTsv.<clinit>(ImportTsv.java:8 1)
log4j:ERROR Either File or DatePattern options are not set for appender [DRFA].
log4j:ERROR setFile(null,true) call failed.
java.io.FileNotFoundException: /var/log/hbase/SecurityAuth.audit (Permission den ied)
at java.io.FileOutputStream.open0(Native Method)
at java.io.FileOutputStream.open(FileOutputStream.java:270)
at java.io.FileOutputStream.<init>(FileOutputStream.java:213)
at java.io.FileOutputStream.<init>(FileOutputStream.java:133)
at org.apache.log4j.FileAppender.setFile(FileAppender.java:294)
at org.apache.log4j.FileAppender.activateOptions(FileAppender.java:165)
at org.apache.log4j.DailyRollingFileAppender.activateOptions(DailyRollin gFileAppender.java:223)
at org.apache.log4j.config.PropertySetter.activate(PropertySetter.java:3 07)
at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.j ava:172)
at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.j ava:104)
at org.apache.log4j.PropertyConfigurator.parseAppender(PropertyConfigura tor.java:842)
at org.apache.log4j.PropertyConfigurator.parseCategory(PropertyConfigura tor.java:768)
at org.apache.log4j.PropertyConfigurator.parseCatsAndRenderers(PropertyC onfigurator.java:672)
at org.apache.log4j.PropertyConfigurator.doConfigure(PropertyConfigurato r.java:516)
at org.apache.log4j.PropertyConfigurator.doConfigure(PropertyConfigurato r.java:580)
at org.apache.log4j.helpers.OptionConverter.selectAndConfigure(OptionCon verter.java:526)
at org.apache.log4j.LogManager.<clinit>(LogManager.java:127)
at org.apache.log4j.Logger.getLogger(Logger.java:104)
at org.apache.commons.logging.impl.Log4JLogger.getLogger(Log4JLogger.jav a:262)
at org.apache.commons.logging.impl.Log4JLogger.<init>(Log4JLogger.java:1 08)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstruct orAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingC onstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.apache.commons.logging.impl.LogFactoryImpl.createLogFromClass(Log FactoryImpl.java:1025)
at org.apache.commons.logging.impl.LogFactoryImpl.discoverLogImplementat ion(LogFactoryImpl.java:844)
at org.apache.commons.logging.impl.LogFactoryImpl.newInstance(LogFactory Impl.java:541)
at org.apache.commons.logging.impl.LogFactoryImpl.getInstance(LogFactory Impl.java:292)
at org.apache.commons.logging.impl.LogFactoryImpl.getInstance(LogFactory Impl.java:269)
at org.apache.commons.logging.LogFactory.getLog(LogFactory.java:655)
at org.apache.hadoop.hbase.mapreduce.ImportTsv.<clinit>(ImportTsv.java:8 1)
log4j:ERROR Either File or DatePattern options are not set for appender [DRFAS].
So, following the tutorial, to actually import the data into hbase I need to run this command 'hbase org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles /storefile flat_table_sf10', but I always get the error:
log4j:ERROR setFile(null,true) call failed.
java.io.FileNotFoundException: /var/log/hbase/hbase.log (Permission denied)
at java.io.FileOutputStream.open0(Native Method)
at java.io.FileOutputStream.open(FileOutputStream.java:270)
at java.io.FileOutputStream.<init>(FileOutputStream.java:213)
at java.io.FileOutputStream.<init>(FileOutputStream.java:133)
at org.apache.log4j.FileAppender.setFile(FileAppender.java:294)
at org.apache.log4j.FileAppender.activateOptions(FileAppender.java:165)
at org.apache.log4j.DailyRollingFileAppender.activateOptions(DailyRollingFileAppender.java:223)
at org.apache.log4j.config.PropertySetter.activate(PropertySetter.java:307)
at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:172)
at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:104)
at org.apache.log4j.PropertyConfigurator.parseAppender(PropertyConfigurator.java:842)
at org.apache.log4j.PropertyConfigurator.parseCategory(PropertyConfigurator.java:768)
at org.apache.log4j.PropertyConfigurator.configureRootCategory(PropertyConfigurator.java:648)
at org.apache.log4j.PropertyConfigurator.doConfigure(PropertyConfigurator.java:514)
at org.apache.log4j.PropertyConfigurator.doConfigure(PropertyConfigurator.java:580)
at org.apache.log4j.helpers.OptionConverter.selectAndConfigure(OptionConverter.java:526)
at org.apache.log4j.LogManager.<clinit>(LogManager.java:127)
at org.apache.log4j.Logger.getLogger(Logger.java:104)
at org.apache.commons.logging.impl.Log4JLogger.getLogger(Log4JLogger.java:262)
at org.apache.commons.logging.impl.Log4JLogger.<init>(Log4JLogger.java:108)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.apache.commons.logging.impl.LogFactoryImpl.createLogFromClass(LogFactoryImpl.java:1025)
at org.apache.commons.logging.impl.LogFactoryImpl.discoverLogImplementation(LogFactoryImpl.java:844)
at org.apache.commons.logging.impl.LogFactoryImpl.newInstance(LogFactoryImpl.java:541)
at org.apache.commons.logging.impl.LogFactoryImpl.getInstance(LogFactoryImpl.java:292)
at org.apache.commons.logging.impl.LogFactoryImpl.getInstance(LogFactoryImpl.java:269)
at org.apache.commons.logging.LogFactory.getLog(LogFactory.java:655)
at org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles.<clinit>(LoadIncrementalHFiles.java:118)
log4j:ERROR Either File or DatePattern options are not set for appender [DRFA].
log4j:ERROR setFile(null,true) call failed.
java.io.FileNotFoundException: /var/log/hbase/SecurityAuth.audit (Permission denied)
at java.io.FileOutputStream.open0(Native Method)
at java.io.FileOutputStream.open(FileOutputStream.java:270)
at java.io.FileOutputStream.<init>(FileOutputStream.java:213)
at java.io.FileOutputStream.<init>(FileOutputStream.java:133)
at org.apache.log4j.FileAppender.setFile(FileAppender.java:294)
at org.apache.log4j.FileAppender.activateOptions(FileAppender.java:165)
at org.apache.log4j.DailyRollingFileAppender.activateOptions(DailyRollingFileAppender.java:223)
at org.apache.log4j.config.PropertySetter.activate(PropertySetter.java:307)
at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:172)
at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:104)
at org.apache.log4j.PropertyConfigurator.parseAppender(PropertyConfigurator.java:842)
at org.apache.log4j.PropertyConfigurator.parseCategory(PropertyConfigurator.java:768)
at org.apache.log4j.PropertyConfigurator.parseCatsAndRenderers(PropertyConfigurator.java:672)
at org.apache.log4j.PropertyConfigurator.doConfigure(PropertyConfigurator.java:516)
at org.apache.log4j.PropertyConfigurator.doConfigure(PropertyConfigurator.java:580)
at org.apache.log4j.helpers.OptionConverter.selectAndConfigure(OptionConverter.java:526)
at org.apache.log4j.LogManager.<clinit>(LogManager.java:127)
at org.apache.log4j.Logger.getLogger(Logger.java:104)
at org.apache.commons.logging.impl.Log4JLogger.getLogger(Log4JLogger.java:262)
at org.apache.commons.logging.impl.Log4JLogger.<init>(Log4JLogger.java:108)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.apache.commons.logging.impl.LogFactoryImpl.createLogFromClass(LogFactoryImpl.java:1025)
at org.apache.commons.logging.impl.LogFactoryImpl.discoverLogImplementation(LogFactoryImpl.java:844)
at org.apache.commons.logging.impl.LogFactoryImpl.newInstance(LogFactoryImpl.java:541)
at org.apache.commons.logging.impl.LogFactoryImpl.getInstance(LogFactoryImpl.java:292)
at org.apache.commons.logging.impl.LogFactoryImpl.getInstance(LogFactoryImpl.java:269)
at org.apache.commons.logging.LogFactory.getLog(LogFactory.java:655)
at org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles.<clinit>(LoadIncrementalHFiles.java:118)
log4j:ERROR Either File or DatePattern options are not set for appender [DRFAS].
I would like to understand how to resolve these errors or if there is a more efficient method of performing this task.
sudo chmod -R 770 /var/log, at least, on whatever server is returning that error. Not really sure you need HDFS, however. Spark could also read from S3 and load into Hbase