在Docker容器中使用Hadoop執行Python MapReduce作業

$ git clone git@github.com:wxw-matt/docker-hadoop.git ~/docker-hadoop

namenode:
    image: wxwmatt/hadoop-namenode:2.1.1-hadoop3.3.1-java8
+   build:
+       context: namenode
+       dockerfile: Dockerfile
    container_name: namenode

FROM wxwmatt/hadoop-base:2.1.1-hadoop3.3.1-java8
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends python3.6
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
HEALTHCHECK CMD curl -f http://localhost:9870/ || exit 1

$ docker-compose up --build -d

# -*-coding:utf-8 -*
import sys
import string
import collections
wordCount = collections.Counter()
for line in sys.stdin:
    # remove leading and trailing whitespace
    line = line.strip()
    # ignore case to avoid duplicates
    lower = line.lower()
    # replace punctuation with spaces, except for hyphens
    punc = string.punctuation
    punc_nohyphens = punc.replace('-', '')
    justText = lower.translate(
        str.maketrans(' ', ' ', punc_nohyphens))
    # allow hyphenated words, but watch out for
    # double-hyphens which might be a stand-in for em-dash
    emdashRemoved = justText.replace('--', ' ')
    # split the line into words
    words = emdashRemoved.split()
    # now we can remove leading and trailing curly quotes 
    # and hyphens, but leave apostrophes in the middle 
    # and allow hyphenated words.
    # note that we lose possessive ending in s, e.g Angus'
    punc_curly = '”’‘“-'
    for word in words:
        word = word.strip(punc_curly)
        if len(word) > 0:
            wordCount[word] += 1
# write the results, tab delimited
# which will be the input for reducer.py
for word, count in wordCount.most_common():
    print ('%s\t%s' % (word, count))

# -*-coding:utf-8 -*
import sys
import collections
import statistics
wordCount = collections.Counter()
wordLengthCount = collections.Counter()
wordLengthList = []
for line in sys.stdin:
    # remove leading and trailing whitespace
    line = line.strip()
    # parse the input we got from mapper.py
    word, count = line.split('\t', 1)
    # ensure that it's a string followed by an int
    try:
        word = str(word)
        count = int(count)
    except ValueError:
        # count was not a number, so silently
        # ignore/discard this line
        continue
    length = len(word)
    wordCount[word] += count
    wordLengthCount[length] += count
    # expand the counted words to a full list
    # to enable median to be calculated later
    lengthRepeated = [length] * count
    wordLengthList.extend(lengthRepeated)
for commonWord, commonWordCount in wordCount.most_common(100):
    print ("{}\t{}".format(commonWord, commonWordCount))
print ("\n")
for commonLength, lengthCount in wordLengthCount.most_common():
    print ("{}\t{}".format(commonLength, commonLengthCount))
print ("\n")
print("Number of words: {}".format(
    len(wordLengthList)))
print("Median word length: {:.0f}".format(
    statistics.median(wordLengthList)))

> hadoop fs -mkdir -p /test/
> hadoop fs -copyFromLocal -f /app/data/input /test/。

> hadoop fs -rm -r -f /test/output

> cd /app/jars
> mapred streaming \
-files mapper.py,reducer.py \
-input /test/input \
-output /test/output \
-mapper “/usr/bin/python mapper.py” \
-reducer “/usr/bin/python reducer.py”

> hadoop fs cat “/test/output/*”