從Spark MLlib到美圖機器學習框架實踐

import org.apache.spark.ml.{Pipeline, PipelineModel
  }import org.apache.spark.ml.classification.LogisticRegressionimport org.apache.spark.ml.feature.{HashingTF, Tokenizer
  }import org.apache.spark.ml.linalg.Vectorimport org.apache.spark.sql.Row// Prepare training documents from a list of (id, text, label) tuples.val training = spark.createDataFrame(Seq( (0L, "a b c d e spark", 1.0), (1L, "b d", 0.0), (2L, "spark f g h", 1.0), (3L, "hadoop mapreduce", 0.0))).toDF("id", "text", "label")// Configure an ML pipeline, which consists of three stages: tokenizer, hashingTF, and lr.val tokenizer = new Tokenizer() .setInputCol("text") .setOutputCol("words")val hashingTF = new HashingTF() .setNumFeatures(1000) .setInputCol(tokenizer.getOutputCol) .setOutputCol("features")val lr = new LogisticRegression() .setMaxIter(10) .setRegParam(0.001)val pipeline = new Pipeline() .setStages(Array(tokenizer, hashingTF, lr))// Fit the pipeline to training documents.val model = pipeline.fit(training)// Now we can optionally save the fitted pipeline to diskmodel.write.overwrite().save("/tmp/spark-logistic-regression-model")// We can also save this unfit pipeline to diskpipeline.write.overwrite().save("/tmp/unfit-lr-model")// And load it back in during productionval sameModel = PipelineModel.load("/tmp/spark-logistic-regression-model")// Prepare test documents, which are unlabeled (id, text) tuples.val test = spark.createDataFrame(Seq( (4L, "spark i j k"), (5L, "l m n"), (6L, "spark hadoop spark"), (7L, "apache hadoop"))).toDF("id", "text")// Make predictions on test documents.model.transform(test) .select("id", "text", "probability", "prediction") .collect() .foreach { 
case Row(id: Long, text: String, prob: Vector, prediction: Double) =>
   println(s"($id, $text) -->
 prob=$prob, prediction=$prediction") 
  }複製程式碼

// We use a ParamGridBuilder to construct a grid of parameters to search over.// With 3 values for hashingTF.numFeatures and 2 values for lr.regParam,// this grid will have 3 x 2 = 6 parameter settings for CrossValidator to choose from.val paramGrid = new ParamGridBuilder() .addGrid(hashingTF.numFeatures, Array(10, 100, 1000)) .addGrid(lr.regParam, Array(0.1, 0.01)) .build()// We now treat the Pipeline as an Estimator, wrapping it in a CrossValidator instance.// This will allow us to jointly choose parameters for all Pipeline stages.// A CrossValidator requires an Estimator, a set of Estimator ParamMaps, and an Evaluator.// Note that the evaluator here is a BinaryClassificationEvaluator and its default metric// is areaUnderROC.val cv = new CrossValidator() .setEstimator(pipeline) .setEvaluator(new BinaryClassificationEvaluator) .setEstimatorParamMaps(paramGrid) .setNumFolds(2)  // Use 3+ in practice .setParallelism(2)  // Evaluate up to 2 parameter settings in parallel// Run cross-validation, and choose the best set of parameters.val cvModel = cv.fit(training)// Prepare test documents, which are unlabeled (id, text) tuples.val test = spark.createDataFrame(Seq( (4L, "spark i j k"), (5L, "l m n"), (6L, "mapreduce spark"), (7L, "apache hadoop"))).toDF("id", "text")// Make predictions on test documents. cvModel uses the best model found (lrModel).cvModel.transform(test) .select("id", "text", "probability", "prediction") .collect() .foreach { 
case Row(id: Long, text: String, prob: Vector, prediction: Double) =>
   println(s"($id, $text) -->
 prob=$prob, prediction=$prediction") 
  }複製程式碼

import org.apache.spark.ml.evaluation.RegressionEvaluatorimport org.apache.spark.ml.regression.LinearRegressionimport org.apache.spark.ml.tuning.{ParamGridBuilder, TrainValidationSplit
  }// Prepare training and test data.val data = spark.read.format("libsvm").load("data/mllib/sample_linear_regression_data.txt")val Array(training, test) = data.randomSplit(Array(0.9, 0.1), seed = 12345)val lr = new LinearRegression()   .setMaxIter(10)// We use a ParamGridBuilder to construct a grid of parameters to search over.// TrainValidationSplit will try all combinations of values and determine best model using// the evaluator.val paramGrid = new ParamGridBuilder() .addGrid(lr.regParam, Array(0.1, 0.01)) .addGrid(lr.fitIntercept) .addGrid(lr.elasticNetParam, Array(0.0, 0.5, 1.0)) .build()// In this case the estimator is simply the linear regression.// A TrainValidationSplit requires an Estimator, a set of Estimator ParamMaps, and an Evaluator.val trainValidationSplit = new TrainValidationSplit() .setEstimator(lr) .setEvaluator(new RegressionEvaluator) .setEstimatorParamMaps(paramGrid) // 80% of the data will be used for training and the remaining 20% for validation. .setTrainRatio(0.8) // Evaluate up to 2 parameter settings in parallel .setParallelism(2)// Run train validation split, and choose the best set of parameters.val model = trainValidationSplit.fit(training)// Make predictions on test data. model is the model with combination of parameters// that performed best.model.transform(test) .select("features", "label", "prediction") .show()複製程式碼