diff options
Diffstat (limited to 'ambari-logsearch')
28 files changed, 903 insertions, 14 deletions
diff --git a/ambari-logsearch/README.md b/ambari-logsearch/README.md index 04fdf2ffd5..23790cddbe 100644 --- a/ambari-logsearch/README.md +++ b/ambari-logsearch/README.md @@ -1,16 +1,36 @@ -# logsearch -RPM/DPKG Build Process -============= +# Log Search -1. Check out the code from GIT repository +## Setup local environment with docker -2. On the logsearch root folder, please execute the following Maven command to build RPM/DPKG: +### Prerequisites - $ mvn -Dbuild-rpm clean package +Install [docker] (https://docs.docker.com/) +For Mac OS X use [Docker Machine] (https://docs.docker.com/machine/) - or +### Build and start Log Search in docker container +```bash +# to see available commands: run start-logsearch without arguments +cd docker +./logsearch-docker build-and-run +``` +If you run the script at first time, it will generate you a new Profile file inside docker directory, in that file you should set AMBARI_LOCATION (point to the local ambari root folder) and MAVEN_REPOSITORY_LOCATION (point to local maven repository location). These will be used as volumes for the docker container. If Profile is generated for first time use `./logsearch-docker start` command to start container without rebuild the project/container again. + +After the logsearch container is started you can enter to it with following command: +```bash +docker exec -it logsearch bash +``` +## Package build process - $ mvn -Dbuild-deb clean package +1. Check out the code from GIT repository + +2. On the logsearch root folder (ambari/ambari-logsearch), please execute the following Maven command to build RPM/DPKG: +```bash +mvn -Dbuild-rpm clean package +``` + or +```bash +mvn -Dbuild-deb clean package +``` 3. Generated RPM/DPKG files will be found in ambari-logsearch-assembly/target folder diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederUtil.java index 15dd5c9584..9881b55782 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederUtil.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederUtil.java @@ -87,6 +87,15 @@ public class LogFeederUtil { return gson; } + private static ThreadLocal<SimpleDateFormat> dateFormatter = new ThreadLocal<SimpleDateFormat>() { + @Override + protected SimpleDateFormat initialValue() { + SimpleDateFormat sdf = new SimpleDateFormat(SOLR_DATE_FORMAT); + sdf.setTimeZone(TimeZone.getTimeZone("UTC")); + return sdf; + } + }; + /** * This method will read the properties from System, followed by propFile * and finally from the map @@ -382,11 +391,18 @@ public class LogFeederUtil { public static String getDate(String timeStampStr) { try { - DateFormat sdf = new SimpleDateFormat(SOLR_DATE_FORMAT); - sdf.setTimeZone(TimeZone.getTimeZone("UTC")); - Date netDate = (new Date(Long.parseLong(timeStampStr))); - return sdf.format(netDate); + return dateFormatter.get().format(new Date(Long.parseLong(timeStampStr))); + } catch (Exception ex) { + logger.error(ex); + return null; + } + } + + public static String getActualDateStr() { + try { + return dateFormatter.get().format(new Date()); } catch (Exception ex) { + logger.error(ex); return null; } } diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java index 43d908e48b..b4dac72398 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java @@ -59,6 +59,7 @@ public class OutputSolr extends Output { private static final int DEFAULT_NUMBER_OF_SHARDS = 1; private static final int DEFAULT_SPLIT_INTERVAL = 30; private static final int DEFAULT_NUMBER_OF_WORKERS = 1; + private static final boolean DEFAULT_SKIP_LOGTIME = false; private static final int RETRY_INTERVAL = 30; @@ -71,6 +72,7 @@ public class OutputSolr extends Output { private int maxBufferSize; private boolean isComputeCurrentCollection = false; private int lastSlotByMin = -1; + private boolean skipLogtime = false; private BlockingQueue<OutputData> outgoingBuffer = null; private List<SolrWorkerThread> workerThreadList = new ArrayList<>(); @@ -96,6 +98,8 @@ public class OutputSolr extends Output { numberOfShards = getIntValue("number_of_shards", DEFAULT_NUMBER_OF_SHARDS); + skipLogtime = getBooleanValue("skip_logtime", DEFAULT_SKIP_LOGTIME); + maxIntervalMS = getIntValue("idle_flush_time_ms", DEFAULT_MAX_INTERVAL_MS); workers = getIntValue("workers", DEFAULT_NUMBER_OF_WORKERS); @@ -243,12 +247,19 @@ public class OutputSolr extends Output { public void write(Map<String, Object> jsonObj, InputMarker inputMarker) throws Exception { try { trimStrValue(jsonObj); + useActualDateIfNeeded(jsonObj); outgoingBuffer.put(new OutputData(jsonObj, inputMarker)); } catch (InterruptedException e) { // ignore } } + private void useActualDateIfNeeded(Map<String, Object> jsonObj) { + if (skipLogtime) { + jsonObj.put("logtime", LogFeederUtil.getActualDateStr()); + } + } + public void flush() { LOG.info("Flush called..."); setDrain(true); diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh index dba85fa279..dfc7b7bb98 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh @@ -59,6 +59,11 @@ if [ "$LOGFEEDER_CONF_DIR" = "" ]; then fi +LOGFEEDER_DEBUG_SUSPEND=${LOGFEEDER_DEBUG_SUSPEND:-n} +if [ "$LOGFEEDER_DEBUG" = "true" ] && [ ! -z "$LOGFEEDER_DEBUG_PORT" ]; then + LOGFEEDER_JAVA_OPTS="$LOGSEARCH_JAVA_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,address=$LOGFEEDER_DEBUG_PORT,server=y,suspend=$LOGFEEDER_DEBUG_SUSPEND " +fi + LOGFEEDER_GC_LOGFILE=`dirname $LOGFILE`/logfeeder_gc.log LOGFEEDER_GC_OPTS="-XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$LOGFEEDER_GC_LOGFILE" diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/scripts/run.sh b/ambari-logsearch/ambari-logsearch-portal/src/main/scripts/run.sh index ca587d5c63..4e8901dc56 100755 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/scripts/run.sh +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/scripts/run.sh @@ -46,11 +46,12 @@ fi #Temporarily enabling JMX so we can monitor the memory and CPU utilization of the process #JMX="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=3098" +LOGSEARCH_DEBUG_SUSPEND=${LOGSEARCH_DEBUG_SUSPEND:-n} if [ "$LOGSEARCH_DEBUG" = "true" ] && [ ! -z "$LOGSEARCH_DEBUG_PORT" ]; then - LOGSEARCH_JAVA_OPTS="$LOGSEARCH_JAVA_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,address=$LOGSEARCH_DEBUG_PORT,server=y,suspend=n " + LOGSEARCH_JAVA_OPTS="$LOGSEARCH_JAVA_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,address=$LOGSEARCH_DEBUG_PORT,server=y,suspend=$LOGSEARCH_DEBUG_SUSPEND " fi -if [ $LOGSEARCH_SSL = "true" ]; then +if [ "$LOGSEARCH_SSL" = "true" ]; then LOGSEARCH_JAVA_OPTS="$LOGSEARCH_JAVA_OPTS -Djavax.net.ssl.keyStore=$LOGSEARCH_KEYSTORE_LOCATION -Djavax.net.ssl.keyStoreType=$LOGSEARCH_KEYSTORE_TYPE -Djavax.net.ssl.keyStorePassword=$LOGSEARCH_KEYSTORE_PASSWORD -Djavax.net.ssl.trustStore=$LOGSEARCH_TRUSTSTORE_LOCATION -Djavax.net.ssl.trustStoreType=$LOGSEARCH_TRUSTSTORE_TYPE -Djavax.net.ssl.trustStorePassword=$LOGSEARCH_TRUSTSTORE_PASSWORD" fi diff --git a/ambari-logsearch/docker/.gitignore b/ambari-logsearch/docker/.gitignore new file mode 100644 index 0000000000..0d6af58052 --- /dev/null +++ b/ambari-logsearch/docker/.gitignore @@ -0,0 +1 @@ +Profile
\ No newline at end of file diff --git a/ambari-logsearch/docker/Dockerfile b/ambari-logsearch/docker/Dockerfile new file mode 100644 index 0000000000..a09f235e7b --- /dev/null +++ b/ambari-logsearch/docker/Dockerfile @@ -0,0 +1,81 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM centos:centos6 + +RUN echo root:changeme | chpasswd + +RUN yum clean all -y && yum update -y +RUN yum -y install vim wget rpm-build sudo which telnet tar openssh-server openssh-clients ntp git python-setuptools python-devel httpd lsof +RUN rpm -e --nodeps --justdb glibc-common +RUN yum -y install glibc-common + +ENV HOME /root + +#Install JAVA +RUN wget --no-check-certificate --no-cookies --header "Cookie:oraclelicense=accept-securebackup-cookie" http://download.oracle.com/otn-pub/java/jdk/7u55-b13/jdk-7u55-linux-x64.rpm -O jdk-7u55-linux-x64.rpm +RUN rpm -ivh jdk-7u55-linux-x64.rpm +ENV JAVA_HOME /usr/java/default/ + +#Install Maven +RUN mkdir -p /opt/maven +WORKDIR /opt/maven +RUN wget http://archive.apache.org/dist/maven/maven-3/3.0.5/binaries/apache-maven-3.0.5-bin.tar.gz +RUN tar -xvzf /opt/maven/apache-maven-3.0.5-bin.tar.gz +RUN rm -rf /opt/maven/apache-maven-3.0.5-bin.tar.gz + +ENV M2_HOME /opt/maven/apache-maven-3.0.5 +ENV MAVEN_OPTS -Xmx2048m -XX:MaxPermSize=256m +ENV PATH $PATH:$JAVA_HOME/bin:$M2_HOME/bin + +# SSH key +RUN ssh-keygen -f /root/.ssh/id_rsa -t rsa -N '' +RUN cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys +RUN chmod 600 /root/.ssh/authorized_keys +RUN sed -ri 's/UsePAM yes/UsePAM no/g' /etc/ssh/sshd_config + +#To allow bower install behind proxy. See https://github.com/bower/bower/issues/731 +RUN git config --global url."https://".insteadOf git:// + +# Install python, nodejs and npm +RUN yum -y install http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm +RUN yum -y install nodejs npm --enablerepo=epel +RUN npm install -g npm@2.1.11 +RUN npm install -g brunch@1.7.20 + +# Install Solr +ENV SOLR_VERSION 5.5.2 +RUN wget --no-check-certificate -O /root/solr-$SOLR_VERSION.tgz http://archive.apache.org/dist/lucene/solr/$SOLR_VERSION/solr-$SOLR_VERSION.tgz +RUN cd /root && tar -zxvf /root/solr-$SOLR_VERSION.tgz +ADD bin/start.sh /root/start.sh +ADD test-config /root/test-config +ADD test-logs /root/test-logs +RUN chmod -R 777 /root/test-config +RUN chmod +x /root/start.sh + +ENV SOLR_CONFIG_LOCATION /root/test-config/solr +ENV LOGSEARCH_CONFIG_LOCATION /root/test-config/logsearch +ENV LOGFEEDER_CONFIG_LOCATION /root/test-config/logfeeder +ENV SOLR_INCLUDE /root/test-config/solr/solr-env.sh +ENV LOGSEARCH_INCLUDE /root/test-config/logsearch/logsearch-env.sh +ENV LOGFEEDER_INCLUDE /root/test-config/logfeeder/logfeeder-env.sh + +RUN mkdir -p /var/run/ambari-logsearch-solr /var/log/ambari-logsearch-solr /var/run/ambari-logsearch-solr-client \ + /var/log/ambari-logsearch-solr-client /root/logsearch_solr_index/data \ + /var/run/ambari-logsearch-portal /var/log/ambari-logsearch-portal \ + /var/run/ambari-logsearch-logfeeder /var/log/ambari-logsearch-logfeeder + +RUN cp /root/test-config/solr/solr.xml /root/logsearch_solr_index/data +RUN cp /root/test-config/solr/zoo.cfg /root/logsearch_solr_index/data + +WORKDIR /root +CMD /root/start.sh
\ No newline at end of file diff --git a/ambari-logsearch/docker/bin/start.sh b/ambari-logsearch/docker/bin/start.sh new file mode 100644 index 0000000000..a09e4b29c7 --- /dev/null +++ b/ambari-logsearch/docker/bin/start.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +AMBARI_PATH=/root/ambari +LOGSEARCH_SERVER_PATH=$AMBARI_PATH/ambari-logsearch/ambari-logsearch-portal/target/package +LOGFEEDER_PATH=$AMBARI_PATH/ambari-logsearch/ambari-logsearch-logfeeder/target/package +SOLR_LOCATION=/root/solr-$SOLR_VERSION.tgz +SOLR_SERVER_LOCATION=/root/solr-$SOLR_VERSION +ZKCLI=$SOLR_SERVER_LOCATION/server/scripts/cloud-scripts/zkcli.sh + +command="$1" + +function build_all() { + echo "build all" + cd $AMBARI_PATH/ambari-logsearch && mvn clean package -DskipTests && mvn -pl ambari-logsearch-logfeeder clean package -DskipTests +} + +function start_solr() { + echo "Starting Solr..." + /root/solr-$SOLR_VERSION/bin/solr start -cloud -s /root/logsearch_solr_index/data -verbose + touch /var/log/ambari-logsearch-solr/solr.log +} + +function start_logsearch() { + echo "Upload configuration sets ..." + $ZKCLI -zkhost localhost:9983 -cmd upconfig -confdir $LOGSEARCH_SERVER_PATH/solr_configsets/audit_logs/conf -confname audit_logs + $ZKCLI -zkhost localhost:9983 -cmd upconfig -confdir $LOGSEARCH_SERVER_PATH/solr_configsets/hadoop_logs/conf -confname hadoop_logs + $ZKCLI -zkhost localhost:9983 -cmd upconfig -confdir $LOGSEARCH_SERVER_PATH/solr_configsets/history/conf -confname history + + cp $LOGSEARCH_CONFIG_LOCATION/logsearch.properties /root/ambari/ambari-logsearch/ambari-logsearch-portal/target/package/classes/logsearch.properties + cp $LOGSEARCH_CONFIG_LOCATION/log4j.xml /root/ambari/ambari-logsearch/ambari-logsearch-portal/target/package/classes/logsearch.properties + $LOGSEARCH_SERVER_PATH/run.sh + touch /var/log/ambari-logsearch-portal/logsearch-app.log +} + +function start_logfeeder() { + cp $LOGFEEDER_CONFIG_LOCATION/logfeeder.properties /root/ambari/ambari-logsearch/ambari-logsearch-logfeeder/target/package/classes/logfeeder.properties + cp $LOGFEEDER_CONFIG_LOCATION/log4j.xml /root/ambari/ambari-logsearch/ambari-logsearch-logfeeder/target/package/classes/log4j.xml + $LOGFEEDER_PATH/run.sh + touch /var/log/ambari-logsearch-logfeeder/logsearch-logfeeder.log +} + +function log() { + component_log=${COMPONENT_LOG:-"logsearch"} + case $component_log in + "logfeeder") + tail -f /var/log/ambari-logsearch-logfeeder/logsearch-logfeeder.log + ;; + "solr") + tail -f /var/log/ambari-logsearch-solr/solr.log + ;; + *) + tail -f /var/log/ambari-logsearch-portal/logsearch-app.log + ;; + esac +} + +start_solr +start_logsearch +start_logfeeder +log + + + diff --git a/ambari-logsearch/docker/logsearch-docker.sh b/ambari-logsearch/docker/logsearch-docker.sh new file mode 100755 index 0000000000..8cea708f9c --- /dev/null +++ b/ambari-logsearch/docker/logsearch-docker.sh @@ -0,0 +1,107 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +sdir="`dirname \"$0\"`" +: ${1:?"argument is missing: (start|stop|build-and-run|build|build-docker-and-run|build-mvn-and-run|build-docker-only|build-mvn-only)"} +command="$1" + +function build_logsearch_project() { + pushd $sdir/../ + mvn clean package -DskipTests + popd +} + +function build_logsearch_container() { + pushd $sdir + docker build -t ambari-logsearch:v1.0 . + popd +} + +function start_logsearch_container() { + setup_profile + source $sdir/Profile + : ${AMBARI_LOCATION:?"Please set the AMBARI_LOCATION in Profile"} + : ${MAVEN_REPOSITORY_LOCATION:?"Please set the MAVEN_REPOSITORY_LOCATION in Profile"} + kill_logsearch_container + echo "Run Log Search container" + docker run -d --name logsearch --hostname logsearch.apache.org \ + -v $AMBARI_LOCATION:/root/ambari -v $MAVEN_REPOSITORY_LOCATION:/root/.m2 $LOGSEARCH_EXPOSED_PORTS $LOGSEARCH_ENV_OPTS $LOGSEARCH_EXTRA_OPTS $LOGSEARCH_VOLUME_OPTS \ + -v $AMBARI_LOCATION/ambari-logsearch/ambari-logsearch-logfeeder/target/classes:/root/ambari/ambari-logsearch/ambari-logsearch-logfeeder/target/package/classes \ + -v $AMBARI_LOCATION/ambari-logsearch/ambari-logsearch-portal/target/classes:/root/ambari/ambari-logsearch/ambari-logsearch-portal/target/package/classes \ + -v $AMBARI_LOCATION/ambari-logsearch/ambari-logsearch-portal/src/main/webapp:/root/ambari/ambari-logsearch/ambari-logsearch-portal/target/package/classes/webapps/app ambari-logsearch:v1.0 + ip_address=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' logsearch) + echo "Log Search container started on $ip_address (for Mac OSX route to boot2docker/docker-machine VM address, e.g.: 'sudo route add -net 172.17.0.0/16 192.168.59.103')" + echo "You can follow Log Search logs with 'docker logs -f logsearch' command" +} + +function setup_profile() { + if [ -f "$sdir/Profile" ]; + then + echo "Profile file exists" + else + echo "Profile does not exist, Creating a new one..." + cat << EOF > $sdir/Profile +AMBARI_LOCATION=$HOME/prj/ambari +MAVEN_REPOSITORY_LOCATION=$HOME/.m2 +LOGSEARCH_EXPOSED_PORTS="-p 8886:8886 -p 61888:61888 -p 5005:5005 -p 5006:5006" +LOGSEARCH_ENV_OPTS="-e LOGFEEDER_DEBUG_SUSPEND=n -e LOGSEARCH_DEBUG_SUSPEND=n -e COMPONENT_LOG=logsearch" +LOGSEARCH_VOLUME_OPTS="-v $AMBARI_LOCATION/ambari-logsearch/docker/test-logs:/root/test-logs -v $AMBARI_LOCATION/ambari-logsearch/docker/test-config:/root/test-config " +LOGSEARCH_EXTRA_OPTS="" +EOF + echo "Profile has been created. Check it out before starting Log Search. ($sdir/Profile)" + exit + fi; +} + +function kill_logsearch_container() { + echo "Try to remove logsearch container if exists..." + docker rm -f logsearch +} + +case $command in + "build-and-run") + build_logsearch_project + build_logsearch_container + start_logsearch_container + ;; + "build") + build_logsearch_project + build_logsearch_container + ;; + "build-docker-and-run") + build_logsearch_container + start_logsearch_container + ;; + "build-mvn-and-run") + build_logsearch_project + start_logsearch_container + ;; + "build-docker-only") + build_logsearch_container + ;; + "build-mvn-only") + build_logsearch_project + ;; + "start") + start_logsearch_container + ;; + "stop") + kill_logsearch_container + ;; + *) + echo "Available commands: (start|stop|build-and-run|build|build-docker-and-run|build-mvn-and-run|build-docker-only|build-mvn-only)" + ;; +esac diff --git a/ambari-logsearch/docker/test-config/logfeeder/log4j.xml b/ambari-logsearch/docker/test-config/logfeeder/log4j.xml new file mode 100644 index 0000000000..52e3e2aacc --- /dev/null +++ b/ambari-logsearch/docker/test-config/logfeeder/log4j.xml @@ -0,0 +1,43 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd"> + +<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/"> + <appender name="console" class="org.apache.log4j.ConsoleAppender"> + <param name="Target" value="System.out" /> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n" /> + </layout> + </appender> + + <appender name="daily_rolling_file" class="org.apache.log4j.DailyRollingFileAppender"> + <param name="file" value="/var/log/ambari-logsearch-logfeeder/logsearch-logfeeder.log" /> + <param name="datePattern" value="'.'yyyy-MM-dd" /> + <param name="append" value="true" /> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n"/> + </layout> + </appender> + + <root> + <priority value="info" /> + <appender-ref ref="daily_rolling_file" /> + <appender-ref ref="console" /> + </root> + +</log4j:configuration> diff --git a/ambari-logsearch/docker/test-config/logfeeder/logfeeder-env.sh b/ambari-logsearch/docker/test-config/logfeeder/logfeeder-env.sh new file mode 100644 index 0000000000..e69b1db90f --- /dev/null +++ b/ambari-logsearch/docker/test-config/logfeeder/logfeeder-env.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +export LOGFEEDER_PATH=/root/ambari/ambari-logsearch/ambari-logsearch-logfeeder/target/package + +export LOGFEEDER_CONF_DIR=/root/test-config/logfeeder + +#Logfile e.g. /var/log/logfeeder.log +export LOGFILE=/var/log/ambari-logsearch-logfeeder/logfeeder.out + +#pid file e.g. /var/run/logfeeder.pid +export PID_FILE=/var/run/ambari-logsearch-logfeeder/logfeeder.pid + +export JAVA_HOME=/usr/java/default + +LOGFEEDER_JAVA_MEM=${LOGFEEDER_JAVA_MEM:-"-Xmx512m"} + +export LOGFEEDER_DEBUG=true + +export LOGFEEDER_DEBUG_PORT=5006
\ No newline at end of file diff --git a/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties b/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties new file mode 100644 index 0000000000..8112462367 --- /dev/null +++ b/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +logfeeder.checkpoint.folder=/root/checkpoints +logfeeder.metrics.collector.hosts= +logfeeder.config.files=shipper-conf/global.config.json,\ + shipper-conf/output.config.json,\ + shipper-conf/input.config-zookeeper.json,\ + shipper-conf/input.config-logsearch.json,\ + shipper-conf/input.config-hst.json +logfeeder.log.filter.enable=true +logfeeder.solr.config.interval=5 +logfeeder.solr.core.config.name=history +logfeeder.solr.zk_connect_string=localhost:9983
\ No newline at end of file diff --git a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/global.config.json b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/global.config.json new file mode 100644 index 0000000000..7ac29dd94d --- /dev/null +++ b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/global.config.json @@ -0,0 +1,11 @@ +{ + "global":{ + "add_fields":{ + "cluster":"cl1" + }, + "source":"file", + "tail":"true", + "gen_event_md5":"true", + "start_position":"beginning" + } +}
\ No newline at end of file diff --git a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-hst.json b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-hst.json new file mode 100644 index 0000000000..4ccef74125 --- /dev/null +++ b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-hst.json @@ -0,0 +1,37 @@ +{ + "input": [ + { + "type": "hst_agent", + "rowtype": "service", + "path": "/root/test-logs/smartsense/hst-agent-test-log.txt" + } + ], + "filter": [ + { + "filter": "grok", + "conditions": { + "fields": { + "type": [ + "hst_agent" + ] + } + }, + "log4j_format": "", + "multiline_pattern": "^(%{CUSTOM_DATESTAMP:logtime})", + "message_pattern": "(?m)^%{CUSTOM_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVAFILE:file}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values": { + "logtime": { + "map_date": { + "date_pattern":"dd MMM yyyy HH:mm:ss" + } + }, + "level": { + "map_fieldvalue": { + "pre_value": "WARNING", + "post_value": "WARN" + } + } + } + } + ] +}
\ No newline at end of file diff --git a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-logsearch.json b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-logsearch.json new file mode 100644 index 0000000000..aae029246f --- /dev/null +++ b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-logsearch.json @@ -0,0 +1,21 @@ +{ + "input": [ + { + "type": "logsearch_app", + "rowtype": "service", + "path": "/root/test-logs/logsearch/logsearch-test-log.json" + } + ], + "filter": [ + { + "filter": "json", + "conditions": { + "fields": { + "type": [ + "logsearch_app" + ] + } + } + } + ] +} diff --git a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-zookeeper.json b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-zookeeper.json new file mode 100644 index 0000000000..122a9e10f6 --- /dev/null +++ b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-zookeeper.json @@ -0,0 +1,31 @@ +{ + "input": [ + { + "type": "zookeeper", + "rowtype": "service", + "path": "/root/test-logs/zookeeper/zookeeper-test-log.txt" + } + ], + "filter": [ + { + "filter": "grok", + "conditions": { + "fields": { + "type": [ + "zookeeper" + ] + } + }, + "log4j_format": "%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n", + "multiline_pattern": "^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern": "(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}-%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\@%{INT:line_number}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values": { + "logtime": { + "map_date": { + "date_pattern": "yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] +}
\ No newline at end of file diff --git a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/output.config.json b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/output.config.json new file mode 100644 index 0000000000..55fd36ce64 --- /dev/null +++ b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/output.config.json @@ -0,0 +1,38 @@ +{ + "output": [ + { + "is_enabled": "true", + "comment": "Output to solr for service logs", + "destination": "solr", + "zk_connect_string": "localhost:9983", + "collection": "hadoop_logs", + "number_of_shards": "3", + "splits_interval_mins": "2", + "skip_logtime": "true", + "conditions": { + "fields": { + "rowtype": [ + "service" + ] + } + } + }, + { + "comment": "Output to solr for audit records", + "is_enabled": "true", + "destination": "solr", + "zk_connect_string": "localhost:9983", + "collection": "audit_logs", + "number_of_shards": "3", + "splits_interval_mins": "2", + "skip_logtime": "true", + "conditions": { + "fields": { + "rowtype": [ + "audit" + ] + } + } + } + ] +}
\ No newline at end of file diff --git a/ambari-logsearch/docker/test-config/logsearch/log4j.xml b/ambari-logsearch/docker/test-config/logsearch/log4j.xml new file mode 100644 index 0000000000..b80824be85 --- /dev/null +++ b/ambari-logsearch/docker/test-config/logsearch/log4j.xml @@ -0,0 +1,44 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + You under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> +<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd"> +<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/"> + <appender name="console" class="org.apache.log4j.ConsoleAppender"> + <param name="Target" value="System.out" /> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n" /> + </layout> + </appender> + + <appender name="rolling_file" class="org.apache.log4j.RollingFileAppender"> + <param name="file" value="/var/log/ambari-logsearch-portal/logsearch-app.log" /> + <param name="Threshold" value="info" /> + <param name="append" value="true" /> + <param name="maxFileSize" value="10MB" /> + <param name="maxBackupIndex" value="10" /> + <layout class="org.apache.log4j.PatternLayout"> + <param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n" /> + </layout> + </appender> + + <!-- Logs to suppress BEGIN --> + <category name="org.apache.solr.common.cloud.ZkStateReader" additivity="false"> + <priority value="error" /> + <appender-ref ref="console" /> + </category> + <!-- Logs to suppress END --> + + <root> + <level value="debug" /> + <appender-ref ref="console" /> + <appender-ref ref="rolling_file" /> + </root> +</log4j:configuration> diff --git a/ambari-logsearch/docker/test-config/logsearch/logsearch-env.sh b/ambari-logsearch/docker/test-config/logsearch/logsearch-env.sh new file mode 100644 index 0000000000..2ad7055025 --- /dev/null +++ b/ambari-logsearch/docker/test-config/logsearch/logsearch-env.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +export LOGSEARCH_PORT=61888 + +# path containing LogSearch.jar file +export LOGSEARCH_PATH=/root/ambari/ambari-logsearch/ambari-logsearch-portal/target/package + +export LOGSEARCH_CONF_DIR=/root/test-config/logsearch + +export LOGFILE=/var/log/ambari-logsearch-portal/logsearch.log + +export PID_FILE=/var/run/ambari-logsearch-portal/logsearch.pid + +export JAVA_HOME=/usr/java/default + +LOGSEARCH_JAVA_MEM=${LOGSEARCH_JAVA_MEM:-"-Xmx1024m"} + +export LOGSEARCH_DEBUG=true + +export LOGSEARCH_DEBUG_PORT=5005
\ No newline at end of file diff --git a/ambari-logsearch/docker/test-config/logsearch/logsearch.properties b/ambari-logsearch/docker/test-config/logsearch/logsearch.properties new file mode 100644 index 0000000000..38d4aad1ec --- /dev/null +++ b/ambari-logsearch/docker/test-config/logsearch/logsearch.properties @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +logsearch.solr.zk_connect_string=localhost:9983 + +# Service Logs +logsearch.solr.collection.service.logs=hadoop_logs + +logsearch.service.logs.split.interval.mins=15 +logsearch.collection.service.logs.numshards=3 +logsearch.collection.service.logs.replication.factor=2 + +logsearch.service.logs.fields=logtime,level,event_count,ip,type,seq_num,path,file,line_number,host,log_message,id + +# Audit logs +logsearch.solr.audit.logs.zk_connect_string=localhost:9983 +logsearch.solr.collection.audit.logs=audit_logs +logsearch.solr.audit.logs.url= + +logsearch.audit.logs.split.interval.mins=15 +logsearch.collection.audit.logs.numshards=3 +logsearch.collection.audit.logs.replication.factor=2 + +# History logs +logsearch.solr.collection.history=history +logsearch.solr.history.config.name=history +logsearch.collection.history.replication.factor=1 + +# Metrics +logsearch.solr.metrics.collector.hosts= +logsearch.solr.jmx.port=18886 + +# Logfeeder Settings + +logsearch.logfeeder.include.default.level=FATAL,ERROR,WARN,INFO,DEBUG,TRACE + +# logsearch-admin.json +logsearch.auth.file.enable=true +logsearch.login.credentials.file=user_pass.json + +logsearch.auth.ldap.enable=false +logsearch.auth.simple.enable=false +logsearch.auth.external_auth.enable=false + +logsearch.protocol=http
\ No newline at end of file diff --git a/ambari-logsearch/docker/test-config/solr/log4j.properties b/ambari-logsearch/docker/test-config/solr/log4j.properties new file mode 100644 index 0000000000..adb2d9ebc7 --- /dev/null +++ b/ambari-logsearch/docker/test-config/solr/log4j.properties @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Logging level +solr.log=/var/log/ambari-logsearch-solr +log4j.rootLogger=INFO, file + +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender + +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x [%X{collection} %X{shard} %X{replica} %X{core}] \u2013 %m%n + +#- size rotation with log cleanup. +log4j.appender.file=org.apache.log4j.RollingFileAppender +log4j.appender.file.MaxFileSize=10MB +log4j.appender.file.MaxBackupIndex=9 + +#- File to log to and log format +log4j.appender.file.File=${solr.log}/solr.log +log4j.appender.file.layout=org.apache.log4j.PatternLayout +log4j.appender.file.layout.ConversionPattern=%d{ISO8601} [%t] %-5p [%X{collection} %X{shard} %X{replica} %X{core}] %C (%F:%L) - %m%n + +log4j.logger.org.apache.zookeeper=WARN +log4j.logger.org.apache.hadoop=WARN + +# set to INFO to enable infostream log messages +log4j.logger.org.apache.solr.update.LoggingInfoStream=OFF
\ No newline at end of file diff --git a/ambari-logsearch/docker/test-config/solr/solr-env.sh b/ambari-logsearch/docker/test-config/solr/solr-env.sh new file mode 100644 index 0000000000..92fa354fc6 --- /dev/null +++ b/ambari-logsearch/docker/test-config/solr/solr-env.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# By default the script will use JAVA_HOME to determine which java +# to use, but you can set a specific path for Solr to use without +# affecting other Java applications on your server/workstation. +SOLR_JAVA_HOME=/usr/java/default + +# Increase Java Min/Max Heap as needed to support your indexing / query needs +SOLR_JAVA_MEM="-Xms1024m -Xmx2048m" + +# Enable verbose GC logging +GC_LOG_OPTS="-verbose:gc -XX:+PrintHeapAtGC -XX:+PrintGCDetails \ +-XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime" + +# These GC settings have shown to work well for a number of common Solr workloads +GC_TUNE="-XX:NewRatio=3 \ +-XX:SurvivorRatio=4 \ +-XX:TargetSurvivorRatio=90 \ +-XX:MaxTenuringThreshold=8 \ +-XX:+UseConcMarkSweepGC \ +-XX:+UseParNewGC \ +-XX:ConcGCThreads=4 -XX:ParallelGCThreads=4 \ +-XX:+CMSScavengeBeforeRemark \ +-XX:PretenureSizeThreshold=64m \ +-XX:+UseCMSInitiatingOccupancyOnly \ +-XX:CMSInitiatingOccupancyFraction=50 \ +-XX:CMSMaxAbortablePrecleanTime=6000 \ +-XX:+CMSParallelRemarkEnabled \ +-XX:+ParallelRefProcEnabled" + +# Set the ZooKeeper connection string if using an external ZooKeeper ensemble +# e.g. host1:2181,host2:2181/chroot +# Leave empty if not using SolrCloud +#ZK_HOST="localhost:9983/ambari-solr" + +# Set the ZooKeeper client timeout (for SolrCloud mode) +ZK_CLIENT_TIMEOUT="60000" + +# By default the start script uses "localhost"; override the hostname here +# for production SolrCloud environments to control the hostname exposed to cluster state +#SOLR_HOST="192.168.1.1" + +# By default the start script uses UTC; override the timezone if needed +#SOLR_TIMEZONE="UTC" + +# Set to true to activate the JMX RMI connector to allow remote JMX client applications +# to monitor the JVM hosting Solr; set to "false" to disable that behavior +# (false is recommended in production environments) +ENABLE_REMOTE_JMX_OPTS="true" + +# The script will use SOLR_PORT+10000 for the RMI_PORT or you can set it here +RMI_PORT=18886 + +# Anything you add to the SOLR_OPTS variable will be included in the java +# start command line as-is, in ADDITION to other options. If you specify the +# -a option on start script, those options will be appended as well. Examples: +#SOLR_OPTS="$SOLR_OPTS -Dsolr.autoSoftCommit.maxTime=3000" +#SOLR_OPTS="$SOLR_OPTS -Dsolr.autoCommit.maxTime=60000" +#SOLR_OPTS="$SOLR_OPTS -Dsolr.clustering.enabled=true" + +# Location where the bin/solr script will save PID files for running instances +# If not set, the script will create PID files in $SOLR_TIP/bin +SOLR_PID_DIR=/var/run/ambari-logsearch-solr + +# Path to a directory where Solr creates index files, the specified directory +# must contain a solr.xml; by default, Solr will use server/solr +SOLR_HOME=/root/logsearch_solr_index/data + +# Solr provides a default Log4J configuration properties file in server/resources +# however, you may want to customize the log settings and file appender location +# so you can point the script to use a different log4j.properties file +LOG4J_PROPS=/root/test-config/solr/log4j.properties + +# Location where Solr should write logs to; should agree with the file appender +# settings in server/resources/log4j.properties +SOLR_LOGS_DIR=/var/log/ambari-logsearch-solr + +# Sets the port Solr binds to, default is 8983 +SOLR_PORT=8886
\ No newline at end of file diff --git a/ambari-logsearch/docker/test-config/solr/solr.xml b/ambari-logsearch/docker/test-config/solr/solr.xml new file mode 100644 index 0000000000..45f9c13763 --- /dev/null +++ b/ambari-logsearch/docker/test-config/solr/solr.xml @@ -0,0 +1,26 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +<solr> + <solrcloud> + <str name="host">${host:}</str> + <int name="hostPort">${jetty.port:}</int> + <str name="hostContext">${hostContext:solr}</str> + <int name="zkClientTimeout">${zkClientTimeout:15000}</int> + <bool name="genericCoreNodeNames">${genericCoreNodeNames:true}</bool> + </solrcloud> +</solr>
\ No newline at end of file diff --git a/ambari-logsearch/docker/test-config/solr/zoo.cfg b/ambari-logsearch/docker/test-config/solr/zoo.cfg new file mode 100644 index 0000000000..48665ce791 --- /dev/null +++ b/ambari-logsearch/docker/test-config/solr/zoo.cfg @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +clientPort=9983 +initLimit=10 +autopurge.purgeInterval=24 +syncLimit=5 +tickTime=2000 +dataDir=/hadoop/zookeeper +autopurge.snapRetainCount=30 +server.1=localhost:2888:3888
\ No newline at end of file diff --git a/ambari-logsearch/docker/test-logs/logsearch/logsearch-test-log.json b/ambari-logsearch/docker/test-logs/logsearch/logsearch-test-log.json new file mode 100644 index 0000000000..708d383991 --- /dev/null +++ b/ambari-logsearch/docker/test-logs/logsearch/logsearch-test-log.json @@ -0,0 +1 @@ +{"level":"WARN","file":"ClientCnxn.java","thread_name":"zkCallback-6-thread-10-SendThread(c6402.ambari.apache.org:2181)","line_number":1102,"log_message":"Session 0x355e0023b38001d for server null, unexpected error, closing socket connection and attempting reconnect\njava.net.SocketException: Network is unreachable\n\tat sun.nio.ch.Net.connect0(Native Method)\n\tat sun.nio.ch.Net.connect(Net.java:454)\n\tat sun.nio.ch.Net.connect(Net.java:446)\n\tat sun.nio.ch.SocketChannelImpl.connect(SocketChannelImpl.java:648)\n\tat org.apache.zookeeper.ClientCnxnSocketNIO.registerAndConnect(ClientCnxnSocketNIO.java:277)\n\tat org.apache.zookeeper.ClientCnxnSocketNIO.connect(ClientCnxnSocketNIO.java:287)\n\tat org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:967)\n\tat org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)\n","logger_name":"org.apache.zookeeper.ClientCnxn","logtime":"1468406756757"}
\ No newline at end of file diff --git a/ambari-logsearch/docker/test-logs/smartsense/hst-agent-test-log.txt b/ambari-logsearch/docker/test-logs/smartsense/hst-agent-test-log.txt new file mode 100644 index 0000000000..8efbe2ecdc --- /dev/null +++ b/ambari-logsearch/docker/test-logs/smartsense/hst-agent-test-log.txt @@ -0,0 +1,4 @@ +18 Jul 2016 17:06:46 DEBUG [MainThread] lock.py:97 - Got the handle of lock file. +18 Jul 2016 17:06:46 INFO [MainThread] security.py:175 - Server certificate not exists, downloading +18 Jul 2016 17:06:46 INFO [MainThread] security.py:188 - Downloading server cert from https://myurl:9440/cert/ca/ +18 Jul 2016 17:06:46 WARNING [MainThread] lock.py:60 - Releasing the lock.
\ No newline at end of file diff --git a/ambari-logsearch/docker/test-logs/zookeeper/zookeeper-test-log.txt b/ambari-logsearch/docker/test-logs/zookeeper/zookeeper-test-log.txt new file mode 100644 index 0000000000..904c420cc8 --- /dev/null +++ b/ambari-logsearch/docker/test-logs/zookeeper/zookeeper-test-log.txt @@ -0,0 +1,3 @@ +2016-07-13 10:45:49,640 - WARN [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181:NIOServerCnxn@362] - Exception causing close of session 0x0 due to java.io.IOException: ZooKeeperServer not running +2016-07-13 10:45:49,640 - INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181:NIOServerCnxn@1007] - Closed socket connection for client /192.168.64.101:39626 (no session established for client) +2016-07-13 10:45:50,351 - INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181:NIOServerCnxnFactory@197] - Accepted socket connection from /192.168.64.101:39632
\ No newline at end of file diff --git a/ambari-logsearch/pom.xml b/ambari-logsearch/pom.xml index 99b2260e67..a8a700bad5 100644 --- a/ambari-logsearch/pom.xml +++ b/ambari-logsearch/pom.xml @@ -158,6 +158,8 @@ <exclude>README.md</exclude> <exclude>**/*.json</exclude> <exclude>**/*.log</exclude> + <exclude>**/*.txt</exclude> + <exclude>**/docker/Profile</exclude> </excludes> </configuration> <executions> |