working PM2 implementation, segmented WE Tools container

docker-wenotes/Dockerfile

@@ -10,7 +10,7 @@
 # License for the specific language governing permissions and limitations under
 # the License.
 
-FROM ubuntu:trusty
+FROM ubuntu:xenial
 
 MAINTAINER Dave Lane dave@oerfoundation.org
 # (adapted from work by Clemens Stolle klaemo@apache.org)
@@ -23,7 +23,7 @@ MAINTAINER Dave Lane dave@oerfoundation.org
 # updating details for session validation, and possibly integrating Mautic...
 ENV APT_SERVER http://ucmirror.canterbury.ac.nz/ubuntu
 ENV APT_FILE sources.list
-ENV UBUNTU_NAME trusty
+ENV UBUNTU_NAME xenial
 # subdue the "Dialog" errors...
 ENV DEBIAN_FRONTEND noninteractive
 #
@@ -43,15 +43,24 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends \
     apt-transport-https \
     apt-utils \
     ca-certificates \
+    cron \
     curl \
-    git \
-    haproxy \
     erlang-nox \
     erlang-reltool \
-    libicu52 \
+    git \
+    haproxy \
+    less \
     libmozjs185-1.0 \
-    openssl
-#  && rm -rf /var/lib/apt/lists/*
+    openssl \
+    python \
+    python-bs4 \
+    python-chardet \
+    python-couchdb \
+    python-feedparser \
+    python-html5lib \
+    python-lxml \
+    python-pip \
+    python-urllib3
 
 # grab gosu for easy step-down from root and tini for signal handling
 RUN gpg --keyserver ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 \
@@ -67,8 +76,6 @@ RUN gpg --keyserver ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364
   && rm /usr/local/bin/tini.asc \
   && chmod +x /usr/local/bin/tini
 
-ENV COUCHDB_VERSION 2.0.0
-
 # install Node.JS repos and do another update...
 RUN curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
   && echo 'deb https://deb.nodesource.com/node_4.x trusty main' > /etc/apt/sources.list.d/nodesource.list \
@@ -76,60 +83,28 @@ RUN curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -
   && apt-get update -y -qq
 # Download dev dependencies
 RUN apt-get install -y --no-install-recommends \
-    build-essential \
-    erlang-dev \
-    libcurl4-openssl-dev \
-    libicu-dev \
-    libmozjs185-dev \
- # install nodejs stack...
- && apt-get install -y nodejs \
- && npm install -g grunt-cli
- # set up pm2
- RUN npm install -g pm2
- # Acquire CouchDB source code
- RUN cd /usr/src && mkdir couchdb \
- && curl -fSL https://dist.apache.org/repos/dist/release/couchdb/source/2.0.0/apache-couchdb-$COUCHDB_VERSION.tar.gz -o couchdb.tar.gz \
- && tar -xzf couchdb.tar.gz -C couchdb --strip-components=1 \
- && cd couchdb \
- # Build the release and install into /opt
- && ./configure --disable-docs \
- && make release \
- && mv /usr/src/couchdb/rel/couchdb /opt/ \
- # Cleanup build detritus
-# && apt-get purge -y \
-#    binutils \
-#    build-essential \
-#    cpp \
-#    erlang-dev \
-#    libicu-dev \
-#    make \
-#    perl \
- && apt-get autoremove -y && apt-get clean \
- && apt-get install -y libicu52 --no-install-recommends \
-# && rm -rf /var/lib/apt/lists/* /usr/lib/node_modules /usr/src/couchdb*
- && rm -rf /usr/src/couchdb*
+  build-essential \
+  erlang-dev \
+  libcurl4-openssl-dev \
+  libmozjs185-dev \
+  # install nodejs stack...
+  && apt-get install -y nodejs \
+  && npm install -g grunt-cli
+# set up pm2
+RUN npm install -g pm2
+# Cleanup build detritus
+RUN apt-get autoremove -y && apt-get clean
 
-
-# Sprinkle couchdb configuration to the right places...
-COPY couchdb/local.ini /opt/couchdb/etc/
-COPY couchdb/vm.args /opt/couchdb/etc/
-COPY couchdb/couchdb-init.sh /
-
-# Setup directories and permissions
-RUN chmod +x /couchdb-init.sh \
- && mkdir /opt/couchdb/data /opt/couchdb/etc/local.d /opt/couchdb/etc/default.d \
- && chown -R couchdb:couchdb /opt/couchdb/
-
- # Set up the language variables
- ENV LANG en_NZ.UTF-8
- ENV LANGUAGE en_NZ.UTF-8
- ENV LC_ALL en_NZ.UTF-8
- # Compile the language spec
- RUN locale-gen $LANG
- #
- # Some final exports to get the environment right...
- RUN echo "export VISIBLE=now" >> /etc/profile \
-   && echo "export TERM=xterm" >> /etc/bash.bashrc
+# Set up the language variables
+ENV LANG en_NZ.UTF-8
+ENV LANGUAGE en_NZ.UTF-8
+ENV LC_ALL en_NZ.UTF-8
+# Compile the language spec
+RUN locale-gen $LANG
+#
+# Some final exports to get the environment right...
+RUN echo "export VISIBLE=now" >> /etc/profile \
+  && echo "export TERM=xterm" >> /etc/bash.bashrc
 
 # set up the various scripts we need to run on the container
 # after it's built...
@@ -138,13 +113,10 @@ COPY ./conf /root/conf/
 RUN chmod u+x /root/conf/*.sh
 
 # expose relevant ports
-EXPOSE 80 5984 4369 9100
-VOLUME ["/opt/couchdb/data", "/opt/wenotes"]
+VOLUME ["/opt/wenotes/tools",  "/opt/wenotes/tools/options.json"]
 
 # First, say we're doing it
 RUN echo "running run.sh - /root/conf/run.sh"
-# Actually do it.
-RUN /root/conf/run.sh
-# Say we're finished
-RUN echo "finished run.sh"
-CMD ["/bin/bash"]
+# Actually do it when we run the container.
+ENTRYPOINT ["tini", "--", "/root/conf/run.sh"]
+CMD ["wenotes"]

docker-wenotes/conf/faye.yml

-apps:
-    - script : 'server.js'
-      name : 'Faye Server'
-      merge_logs : true
-      out_file : '/tmp/server.log'

docker-wenotes/conf/options.json.sample

-{
-   "server": {
-    "ip": "localhost",
-    "port": 80,
-    "secureport": "",
-    "privkey": "",
-    "fullchain": ""
-  },
-  "log": {
-	  "couchwatch": {
-		  "level": 7,
-		  "file": "/tmp/couchwatch.log"
-	  },
-	  "twitters": {
-		  "level": 7,
-		  "file": "/tmp/twitters.log"
-	  },
-	  "server": {
-		  "level": 7,
-		  "file": "/tmp/wenotes-server.log"
-	  }
-  }
-}

docker-wenotes/conf/run.sh

@@ -9,12 +9,12 @@ COUCH=/root
 WENOTES=/opt/wenotes
 CWD=`pwd`
 # Defines
-WESERVER=https://kiwilightweight@bitbucket.org/wikieducator/wenotes-server.git
 WETOOLS=https://kiwilightweight@bitbucket.org/wikieducator/wenotes-tools.git
 GIT=`which git`
 NPM=`which npm`
 PM2=`which pm2`
 CP=`which cp`
+CRON=/etc/cron.d/wenotes
 
 echo "CWD=$CWD, GIT=$GIT"
 
@@ -28,7 +28,6 @@ if [[ -d $CONF/before-start ]] ; then
   done
 fi
 
-
 if [[ -f $CONF/pre-install.sh ]] ; then
   echo "Running: pre-install.sh"
   source $CONF/pre-install.sh
@@ -36,44 +35,39 @@ fi
 
 echo "starting services"
 
-# first start couchdb
-if [[ -f $COUCH/couchdb-init.sh ]] ; then
-    echo "starting couchdb via $COUCH/couchdb-init.sh"
-    $PM2 start --no-daemon $COUCH/couchdb-init.sh
-fi
-
 # next, get the Javascript code:
 # get the repo
 echo "moving to $WENOTES"
-cd $WENOTES
-echo "getting $WESERVER, putting it into server"
-$GIT clone $WESERVER server
 # get the repo
 echo "getting $WETOOLS, putting it into server"
+cd /tmp
 $GIT clone $WETOOLS tools
+cd $WENOTES
+if [[ -d $WENOTES/tools ]] ; then
+    echo "$WENOTES/tools already exists - moving /tmp/tools/* there..."
+    cp -a /tmp/tools/* $WENOTES/tools
+else
+    echo "creating $WENOTES/tools"
+    mv /tmp/tools $WENOTES
+fi
 # set up options.json
 
-
-# next start various Javascript services
-if [[ -f $CONF/faye.yml ]] ; then
-    echo "moving to server"
-    cd $WENOTES/server
-    echo "installing Node.JS dependencies"
-    $NPM install
-    $CP $CONF/options.json.sample options.json
-    echo "starting pm2 to supervise scripts in $CONF/faye.yml"
-    $PM2 start --no-daemon $CONF/faye.yml
-    cd $WENOTES
-fi
+# set up Cron jobs...
+echo "setting up cron jobs"
+echo "# created by Docker and the OER Foundation" > $CRON
+echo "4,24,44 * * * * root cd $WENOTES/tools/ && nice python gplus.py && nice python feeds.py && nice python groups.py" >> $CRON
+echo "2,22,42 * * * * root cd $WENOTES/tools/ && nice python forums.py && nice python discourse.py --full " >> $CRON
+chmod 0644 $CRON
 
 # next start various Javascript services
 if [[ -f $CONF/services.yml ]] ; then
     cd $WENOTES/tools
     echo "installing Node.JS dependencies"
     $NPM install
-    $CP $CONF/options.json.sample options.json
+    #$CP $CONF/options.json.sample options.json
     echo "starting pm2 to supervise scripts in $CONF/services.yml"
     $PM2 start --no-daemon $CONF/services.yml
+    #$PM2 start $CONF/services.yml
     cd $WENOTES
 fi