我将下面粘贴在 python 中的现有代码转换为 pyspark。

Python代码：

import json
import csv


def main():
    # create a simple JSON array
    with open('paytm_tweets_data_1495614657.json') as str:

        tweetsList = []
        # change the JSON string into a JSON object
        jsonObject = json.load(str)

        #print(jsonObject)

        # # print the keys and values
        for i in range(len(jsonObject)):
            tweetsList.insert(i,jsonObject[i]["text"])

        #print(tweetsList)
    displaySentiment(tweetsList)



def displaySentiment(tweetsList):
    aDict = {}

    from sentiment import sentiment_score

    for i in range(len(tweetsList)):
        aDict[tweetsList[i]] = sentiment_score(tweetsList[i])
    print (aDict)


    with open('PaytmtweetSentiment.csv', 'w') as csv_file:
        writer = csv.DictWriter(csv_file, fieldnames = ["Tweets", "Sentiment Value"])
        writer.writeheader()
        writer = csv.writer(csv_file)
        for key, value in aDict.items():
            writer.writerow([key, value])


if __name__ == '__main__':
    main()

转换后的 Pyspark 代码：

import json
import csv
import os
from pyspark import SparkContext, SparkConf
from pyspark.python.pyspark.shell import spark

os.environ['PYSPARK_PYTHON'] = "/usr/local/bin/python3"


def main():
    path = "/Users/i322865/DeepInsights/bitbucket-code/ai-engine/twitter-sentiment-analysis/flipkart_tweets_data_1495601666.json"
    peopleDF = spark.read.json(path).rdd
    df = peopleDF.map(lambda row: row['text'])
    print(df.collect())
    displaySentiment(df.collect())



def displaySentiment(tweetsList):
    from sentiment import sentiment_score

    aDict = sentiment_score(tweetsList)

    #
    with open('paytmtweetSentiment.csv', 'w') as csv_file:
        writer = csv.DictWriter(csv_file, fieldnames = ["Tweets", "Sentiment Value"])
        writer.writeheader()
        writer = csv.writer(csv_file)
        for i in range(len(tweetsList)):
            writer.writerow([tweetsList[i], aDict[i]])
            print([tweetsList[i], aDict[i]])


if __name__ == '__main__':
    conf = SparkConf().setAppName("Test").setMaster("local")
    sc = SparkContext.getOrCreate(conf=conf)
    main()

我运行了这两个程序，但没有看到任何显着的性能改进。我错过了什么？请问你能发表一些想法吗？