## Licensed to the Apache Software Foundation (ASF) under one or more# contributor license agreements. See the NOTICE file distributed with# this work for additional information regarding copyright ownership.# The ASF licenses this file to You under the Apache License, Version 2.0# (the "License"); you may not use this file except in compliance with# the License. You may obtain a copy of the License at## http://www.apache.org/licenses/LICENSE-2.0## Unless required by applicable law or agreed to in writing, software# distributed under the License is distributed on an "AS IS" BASIS,# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.#importsysfromtypingimportAny,Dict,Generic,List,Optional,TypeVar,TYPE_CHECKINGfromabcimportABCMetafrompysparkimportkeyword_only,sincefrompyspark.mlimportPredictor,PredictionModelfrompyspark.ml.baseimport_PredictorParamsfrompyspark.ml.param.sharedimport(HasFeaturesCol,HasLabelCol,HasPredictionCol,HasWeightCol,Param,Params,TypeConverters,HasMaxIter,HasTol,HasFitIntercept,HasAggregationDepth,HasMaxBlockSizeInMB,HasRegParam,HasSolver,HasStepSize,HasSeed,HasElasticNetParam,HasStandardization,HasLoss,HasVarianceCol,)frompyspark.ml.treeimport(_DecisionTreeModel,_DecisionTreeParams,_TreeEnsembleModel,_RandomForestParams,_GBTParams,_TreeRegressorParams,)frompyspark.ml.baseimportTransformerfrompyspark.ml.linalgimportVector,Matrixfrompyspark.ml.utilimport(JavaMLWritable,JavaMLReadable,HasTrainingSummary,GeneralJavaMLWritable,)frompyspark.ml.wrapperimport(JavaEstimator,JavaModel,JavaPredictor,JavaPredictionModel,JavaTransformer,JavaWrapper,)frompyspark.ml.commonimportinherit_docfrompyspark.sqlimportDataFrameifTYPE_CHECKING:frompy4j.java_gatewayimportJavaObjectT=TypeVar("T")M=TypeVar("M",bound=Transformer)JM=TypeVar("JM",bound=JavaTransformer)__all__=["AFTSurvivalRegression","AFTSurvivalRegressionModel","DecisionTreeRegressor","DecisionTreeRegressionModel","GBTRegressor","GBTRegressionModel","GeneralizedLinearRegression","GeneralizedLinearRegressionModel","GeneralizedLinearRegressionSummary","GeneralizedLinearRegressionTrainingSummary","IsotonicRegression","IsotonicRegressionModel","LinearRegression","LinearRegressionModel","LinearRegressionSummary","LinearRegressionTrainingSummary","RandomForestRegressor","RandomForestRegressionModel","FMRegressor","FMRegressionModel",]classRegressor(Predictor[M],_PredictorParams,Generic[M],metaclass=ABCMeta):""" Regressor for regression tasks. .. versionadded:: 3.0.0 """passclassRegressionModel(PredictionModel[T],_PredictorParams,metaclass=ABCMeta):""" Model produced by a ``Regressor``. .. versionadded:: 3.0.0 """passclass_JavaRegressor(Regressor,JavaPredictor[JM],Generic[JM],metaclass=ABCMeta):""" Java Regressor for regression tasks. .. versionadded:: 3.0.0 """passclass_JavaRegressionModel(RegressionModel,JavaPredictionModel[T],metaclass=ABCMeta):""" Java Model produced by a ``_JavaRegressor``. To be mixed in with :class:`pyspark.ml.JavaModel` .. versionadded:: 3.0.0 """passclass_LinearRegressionParams(_PredictorParams,HasRegParam,HasElasticNetParam,HasMaxIter,HasTol,HasFitIntercept,HasStandardization,HasWeightCol,HasSolver,HasAggregationDepth,HasLoss,HasMaxBlockSizeInMB,):""" Params for :py:class:`LinearRegression` and :py:class:`LinearRegressionModel`. .. versionadded:: 3.0.0 """solver:Param[str]=Param(Params._dummy(),"solver","The solver algorithm for optimization. Supported "+"options: auto, normal, l-bfgs.",typeConverter=TypeConverters.toString,)loss:Param[str]=Param(Params._dummy(),"loss","The loss function to be optimized. Supported "+"options: squaredError, huber.",typeConverter=TypeConverters.toString,)epsilon:Param[float]=Param(Params._dummy(),"epsilon","The shape parameter to control the amount of "+"robustness. Must be > 1.0. Only valid when loss is huber",typeConverter=TypeConverters.toFloat,)def__init__(self,*args:Any):super(_LinearRegressionParams,self).__init__(*args)self._setDefault(maxIter=100,regParam=0.0,tol=1e-6,loss="squaredError",epsilon=1.35,maxBlockSizeInMB=0.0,)@since("2.3.0")defgetEpsilon(self)->float:""" Gets the value of epsilon or its default value. """returnself.getOrDefault(self.epsilon)
[docs]@inherit_docclassLinearRegression(_JavaRegressor["LinearRegressionModel"],_LinearRegressionParams,JavaMLWritable,JavaMLReadable["LinearRegression"],):""" Linear regression. The learning objective is to minimize the specified loss function, with regularization. This supports two kinds of loss: * squaredError (a.k.a squared loss) * huber (a hybrid of squared error for relatively small errors and absolute error for \ relatively large ones, and we estimate the scale parameter from training data) This supports multiple types of regularization: * none (a.k.a. ordinary least squares) * L2 (ridge regression) * L1 (Lasso) * L2 + L1 (elastic net) .. versionadded:: 1.4.0 Notes ----- Fitting with huber loss only supports none and L2 regularization. Examples -------- >>> from pyspark.ml.linalg import Vectors >>> df = spark.createDataFrame([ ... (1.0, 2.0, Vectors.dense(1.0)), ... (0.0, 2.0, Vectors.sparse(1, [], []))], ["label", "weight", "features"]) >>> lr = LinearRegression(regParam=0.0, solver="normal", weightCol="weight") >>> lr.setMaxIter(5) LinearRegression... >>> lr.getMaxIter() 5 >>> lr.setRegParam(0.1) LinearRegression... >>> lr.getRegParam() 0.1 >>> lr.setRegParam(0.0) LinearRegression... >>> model = lr.fit(df) >>> model.setFeaturesCol("features") LinearRegressionModel... >>> model.setPredictionCol("newPrediction") LinearRegressionModel... >>> model.getMaxIter() 5 >>> model.getMaxBlockSizeInMB() 0.0 >>> test0 = spark.createDataFrame([(Vectors.dense(-1.0),)], ["features"]) >>> abs(model.predict(test0.head().features) - (-1.0)) < 0.001 True >>> abs(model.transform(test0).head().newPrediction - (-1.0)) < 0.001 True >>> abs(model.coefficients[0] - 1.0) < 0.001 True >>> abs(model.intercept - 0.0) < 0.001 True >>> test1 = spark.createDataFrame([(Vectors.sparse(1, [0], [1.0]),)], ["features"]) >>> abs(model.transform(test1).head().newPrediction - 1.0) < 0.001 True >>> lr.setParams(featuresCol="vector") LinearRegression... >>> lr_path = temp_path + "/lr" >>> lr.save(lr_path) >>> lr2 = LinearRegression.load(lr_path) >>> lr2.getMaxIter() 5 >>> model_path = temp_path + "/lr_model" >>> model.save(model_path) >>> model2 = LinearRegressionModel.load(model_path) >>> model.coefficients[0] == model2.coefficients[0] True >>> model.intercept == model2.intercept True >>> model.transform(test0).take(1) == model2.transform(test0).take(1) True >>> model.numFeatures 1 >>> model.write().format("pmml").save(model_path + "_2") """_input_kwargs:Dict[str,Any]@keyword_onlydef__init__(self,*,featuresCol:str="features",labelCol:str="label",predictionCol:str="prediction",maxIter:int=100,regParam:float=0.0,elasticNetParam:float=0.0,tol:float=1e-6,fitIntercept:bool=True,standardization:bool=True,solver:str="auto",weightCol:Optional[str]=None,aggregationDepth:int=2,loss:str="squaredError",epsilon:float=1.35,maxBlockSizeInMB:float=0.0,):""" __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ maxIter=100, regParam=0.0, elasticNetParam=0.0, tol=1e-6, fitIntercept=True, \ standardization=True, solver="auto", weightCol=None, aggregationDepth=2, \ loss="squaredError", epsilon=1.35, maxBlockSizeInMB=0.0) """super(LinearRegression,self).__init__()self._java_obj=self._new_java_obj("org.apache.spark.ml.regression.LinearRegression",self.uid)kwargs=self._input_kwargsself.setParams(**kwargs)
[docs]@since("2.3.0")defsetEpsilon(self,value:float)->"LinearRegression":""" Sets the value of :py:attr:`epsilon`. """returnself._set(epsilon=value)
[docs]defsetMaxIter(self,value:int)->"LinearRegression":""" Sets the value of :py:attr:`maxIter`. """returnself._set(maxIter=value)
[docs]defsetRegParam(self,value:float)->"LinearRegression":""" Sets the value of :py:attr:`regParam`. """returnself._set(regParam=value)
[docs]defsetTol(self,value:float)->"LinearRegression":""" Sets the value of :py:attr:`tol`. """returnself._set(tol=value)
[docs]defsetElasticNetParam(self,value:float)->"LinearRegression":""" Sets the value of :py:attr:`elasticNetParam`. """returnself._set(elasticNetParam=value)
[docs]defsetFitIntercept(self,value:bool)->"LinearRegression":""" Sets the value of :py:attr:`fitIntercept`. """returnself._set(fitIntercept=value)
[docs]defsetStandardization(self,value:bool)->"LinearRegression":""" Sets the value of :py:attr:`standardization`. """returnself._set(standardization=value)
[docs]defsetWeightCol(self,value:str)->"LinearRegression":""" Sets the value of :py:attr:`weightCol`. """returnself._set(weightCol=value)
[docs]defsetSolver(self,value:str)->"LinearRegression":""" Sets the value of :py:attr:`solver`. """returnself._set(solver=value)
[docs]defsetAggregationDepth(self,value:int)->"LinearRegression":""" Sets the value of :py:attr:`aggregationDepth`. """returnself._set(aggregationDepth=value)
[docs]defsetLoss(self,value:str)->"LinearRegression":""" Sets the value of :py:attr:`loss`. """returnself._set(lossType=value)
[docs]@since("3.1.0")defsetMaxBlockSizeInMB(self,value:float)->"LinearRegression":""" Sets the value of :py:attr:`maxBlockSizeInMB`. """returnself._set(maxBlockSizeInMB=value)
[docs]classLinearRegressionModel(_JavaRegressionModel,_LinearRegressionParams,GeneralJavaMLWritable,JavaMLReadable["LinearRegressionModel"],HasTrainingSummary["LinearRegressionSummary"],):""" Model fitted by :class:`LinearRegression`. .. versionadded:: 1.4.0 """@property# type: ignore[misc]@since("2.0.0")defcoefficients(self)->Vector:""" Model coefficients. """returnself._call_java("coefficients")@property# type: ignore[misc]@since("1.4.0")defintercept(self)->float:""" Model intercept. """returnself._call_java("intercept")@property# type: ignore[misc]@since("2.3.0")defscale(self)->float:r""" The value by which :math:`\|y - X'w\|` is scaled down when loss is "huber", otherwise 1.0. """returnself._call_java("scale")@property# type: ignore[misc]@since("2.0.0")defsummary(self)->"LinearRegressionTrainingSummary":""" Gets summary (residuals, MSE, r-squared ) of model on training set. An exception is thrown if `trainingSummary is None`. """ifself.hasSummary:returnLinearRegressionTrainingSummary(super(LinearRegressionModel,self).summary)else:raiseRuntimeError("No training summary available for this %s"%self.__class__.__name__)
[docs]defevaluate(self,dataset:DataFrame)->"LinearRegressionSummary":""" Evaluates the model on a test dataset. .. versionadded:: 2.0.0 Parameters ---------- dataset : :py:class:`pyspark.sql.DataFrame` Test dataset to evaluate model on, where dataset is an instance of :py:class:`pyspark.sql.DataFrame` """ifnotisinstance(dataset,DataFrame):raiseTypeError("dataset must be a DataFrame but got %s."%type(dataset))java_lr_summary=self._call_java("evaluate",dataset)returnLinearRegressionSummary(java_lr_summary)
[docs]classLinearRegressionSummary(JavaWrapper):""" Linear regression results evaluated on a dataset. .. versionadded:: 2.0.0 """@property# type: ignore[misc]@since("2.0.0")defpredictions(self)->DataFrame:""" Dataframe outputted by the model's `transform` method. """returnself._call_java("predictions")@property# type: ignore[misc]@since("2.0.0")defpredictionCol(self)->str:""" Field in "predictions" which gives the predicted value of the label at each instance. """returnself._call_java("predictionCol")@property# type: ignore[misc]@since("2.0.0")deflabelCol(self)->str:""" Field in "predictions" which gives the true label of each instance. """returnself._call_java("labelCol")@property# type: ignore[misc]@since("2.0.0")deffeaturesCol(self)->str:""" Field in "predictions" which gives the features of each instance as a vector. """returnself._call_java("featuresCol")@property# type: ignore[misc]@since("2.0.0")defexplainedVariance(self)->float:r""" Returns the explained variance regression score. explainedVariance = :math:`1 - \frac{variance(y - \hat{y})}{variance(y)}` Notes ----- This ignores instance weights (setting all to 1.0) from `LinearRegression.weightCol`. This will change in later Spark versions. For additional information see `Explained variation on Wikipedia \ <http://en.wikipedia.org/wiki/Explained_variation>`_ """returnself._call_java("explainedVariance")@property# type: ignore[misc]@since("2.0.0")defmeanAbsoluteError(self)->float:""" Returns the mean absolute error, which is a risk function corresponding to the expected value of the absolute error loss or l1-norm loss. Notes ----- This ignores instance weights (setting all to 1.0) from `LinearRegression.weightCol`. This will change in later Spark versions. """returnself._call_java("meanAbsoluteError")@property# type: ignore[misc]@since("2.0.0")defmeanSquaredError(self)->float:""" Returns the mean squared error, which is a risk function corresponding to the expected value of the squared error loss or quadratic loss. Notes ----- This ignores instance weights (setting all to 1.0) from `LinearRegression.weightCol`. This will change in later Spark versions. """returnself._call_java("meanSquaredError")@property# type: ignore[misc]@since("2.0.0")defrootMeanSquaredError(self)->float:""" Returns the root mean squared error, which is defined as the square root of the mean squared error. Notes ----- This ignores instance weights (setting all to 1.0) from `LinearRegression.weightCol`. This will change in later Spark versions. """returnself._call_java("rootMeanSquaredError")@property# type: ignore[misc]@since("2.0.0")defr2(self)->float:""" Returns R^2, the coefficient of determination. Notes ----- This ignores instance weights (setting all to 1.0) from `LinearRegression.weightCol`. This will change in later Spark versions. See also `Wikipedia coefficient of determination \ <http://en.wikipedia.org/wiki/Coefficient_of_determination>`_ """returnself._call_java("r2")@property# type: ignore[misc]@since("2.4.0")defr2adj(self)->float:""" Returns Adjusted R^2, the adjusted coefficient of determination. Notes ----- This ignores instance weights (setting all to 1.0) from `LinearRegression.weightCol`. This will change in later Spark versions. `Wikipedia coefficient of determination, Adjusted R^2 \ <https://en.wikipedia.org/wiki/Coefficient_of_determination#Adjusted_R2>`_ """returnself._call_java("r2adj")@property# type: ignore[misc]@since("2.0.0")defresiduals(self)->DataFrame:""" Residuals (label - predicted value) """returnself._call_java("residuals")@property# type: ignore[misc]@since("2.0.0")defnumInstances(self)->int:""" Number of instances in DataFrame predictions """returnself._call_java("numInstances")@property# type: ignore[misc]@since("2.2.0")defdegreesOfFreedom(self)->int:""" Degrees of freedom. """returnself._call_java("degreesOfFreedom")@property# type: ignore[misc]@since("2.0.0")defdevianceResiduals(self)->List[float]:""" The weighted residuals, the usual residuals rescaled by the square root of the instance weights. """returnself._call_java("devianceResiduals")@propertydefcoefficientStandardErrors(self)->List[float]:""" Standard error of estimated coefficients and intercept. This value is only available when using the "normal" solver. If :py:attr:`LinearRegression.fitIntercept` is set to True, then the last element returned corresponds to the intercept. .. versionadded:: 2.0.0 See Also -------- LinearRegression.solver """returnself._call_java("coefficientStandardErrors")@propertydeftValues(self)->List[float]:""" T-statistic of estimated coefficients and intercept. This value is only available when using the "normal" solver. If :py:attr:`LinearRegression.fitIntercept` is set to True, then the last element returned corresponds to the intercept. .. versionadded:: 2.0.0 See Also -------- LinearRegression.solver """returnself._call_java("tValues")@propertydefpValues(self)->List[float]:""" Two-sided p-value of estimated coefficients and intercept. This value is only available when using the "normal" solver. If :py:attr:`LinearRegression.fitIntercept` is set to True, then the last element returned corresponds to the intercept. .. versionadded:: 2.0.0 See Also -------- LinearRegression.solver """returnself._call_java("pValues")
[docs]@inherit_docclassLinearRegressionTrainingSummary(LinearRegressionSummary):""" Linear regression training results. Currently, the training summary ignores the training weights except for the objective trace. .. versionadded:: 2.0.0 """@propertydefobjectiveHistory(self)->List[float]:""" Objective function (scaled loss + regularization) at each iteration. This value is only available when using the "l-bfgs" solver. .. versionadded:: 2.0.0 See Also -------- LinearRegression.solver """returnself._call_java("objectiveHistory")@propertydeftotalIterations(self)->int:""" Number of training iterations until termination. This value is only available when using the "l-bfgs" solver. .. versionadded:: 2.0.0 See Also -------- LinearRegression.solver """returnself._call_java("totalIterations")
class_IsotonicRegressionParams(HasFeaturesCol,HasLabelCol,HasPredictionCol,HasWeightCol):""" Params for :py:class:`IsotonicRegression` and :py:class:`IsotonicRegressionModel`. .. versionadded:: 3.0.0 """isotonic:Param[bool]=Param(Params._dummy(),"isotonic","whether the output sequence should be isotonic/increasing (true) or"+"antitonic/decreasing (false).",typeConverter=TypeConverters.toBoolean,)featureIndex:Param[int]=Param(Params._dummy(),"featureIndex","The index of the feature if featuresCol is a vector column, no effect otherwise.",typeConverter=TypeConverters.toInt,)def__init__(self,*args:Any):super(_IsotonicRegressionParams,self).__init__(*args)self._setDefault(isotonic=True,featureIndex=0)defgetIsotonic(self)->bool:""" Gets the value of isotonic or its default value. """returnself.getOrDefault(self.isotonic)defgetFeatureIndex(self)->int:""" Gets the value of featureIndex or its default value. """returnself.getOrDefault(self.featureIndex)
[docs]defsetIsotonic(self,value:bool)->"IsotonicRegression":""" Sets the value of :py:attr:`isotonic`. """returnself._set(isotonic=value)
[docs]defsetFeatureIndex(self,value:int)->"IsotonicRegression":""" Sets the value of :py:attr:`featureIndex`. """returnself._set(featureIndex=value)
[docs]@since("1.6.0")defsetFeaturesCol(self,value:str)->"IsotonicRegression":""" Sets the value of :py:attr:`featuresCol`. """returnself._set(featuresCol=value)
[docs]@since("1.6.0")defsetPredictionCol(self,value:str)->"IsotonicRegression":""" Sets the value of :py:attr:`predictionCol`. """returnself._set(predictionCol=value)
[docs]@since("1.6.0")defsetLabelCol(self,value:str)->"IsotonicRegression":""" Sets the value of :py:attr:`labelCol`. """returnself._set(labelCol=value)
[docs]@since("1.6.0")defsetWeightCol(self,value:str)->"IsotonicRegression":""" Sets the value of :py:attr:`weightCol`. """returnself._set(weightCol=value)
[docs]classIsotonicRegressionModel(JavaModel,_IsotonicRegressionParams,JavaMLWritable,JavaMLReadable["IsotonicRegressionModel"],):""" Model fitted by :class:`IsotonicRegression`. .. versionadded:: 1.6.0 """
[docs]@since("3.0.0")defsetFeaturesCol(self,value:str)->"IsotonicRegressionModel":""" Sets the value of :py:attr:`featuresCol`. """returnself._set(featuresCol=value)
[docs]@since("3.0.0")defsetPredictionCol(self,value:str)->"IsotonicRegressionModel":""" Sets the value of :py:attr:`predictionCol`. """returnself._set(predictionCol=value)
[docs]defsetFeatureIndex(self,value:int)->"IsotonicRegressionModel":""" Sets the value of :py:attr:`featureIndex`. """returnself._set(featureIndex=value)
@property# type: ignore[misc]@since("1.6.0")defboundaries(self)->Vector:""" Boundaries in increasing order for which predictions are known. """returnself._call_java("boundaries")@property# type: ignore[misc]@since("1.6.0")defpredictions(self)->Vector:""" Predictions associated with the boundaries at the same index, monotone because of isotonic regression. """returnself._call_java("predictions")@property# type: ignore[misc]@since("3.0.0")defnumFeatures(self)->int:""" Returns the number of features the model was trained on. If unknown, returns -1 """returnself._call_java("numFeatures")
[docs]@since("3.0.0")defpredict(self,value:float)->float:""" Predict label for the given features. """returnself._call_java("predict",value)
class_DecisionTreeRegressorParams(_DecisionTreeParams,_TreeRegressorParams,HasVarianceCol):""" Params for :py:class:`DecisionTreeRegressor` and :py:class:`DecisionTreeRegressionModel`. .. versionadded:: 3.0.0 """def__init__(self,*args:Any):super(_DecisionTreeRegressorParams,self).__init__(*args)self._setDefault(maxDepth=5,maxBins=32,minInstancesPerNode=1,minInfoGain=0.0,maxMemoryInMB=256,cacheNodeIds=False,checkpointInterval=10,impurity="variance",leafCol="",minWeightFractionPerNode=0.0,)
[docs]@since("1.4.0")defsetMaxDepth(self,value:int)->"DecisionTreeRegressor":""" Sets the value of :py:attr:`maxDepth`. """returnself._set(maxDepth=value)
[docs]@since("1.4.0")defsetMaxBins(self,value:int)->"DecisionTreeRegressor":""" Sets the value of :py:attr:`maxBins`. """returnself._set(maxBins=value)
[docs]@since("1.4.0")defsetMinInstancesPerNode(self,value:int)->"DecisionTreeRegressor":""" Sets the value of :py:attr:`minInstancesPerNode`. """returnself._set(minInstancesPerNode=value)
[docs]@since("3.0.0")defsetMinWeightFractionPerNode(self,value:float)->"DecisionTreeRegressor":""" Sets the value of :py:attr:`minWeightFractionPerNode`. """returnself._set(minWeightFractionPerNode=value)
[docs]@since("1.4.0")defsetMinInfoGain(self,value:float)->"DecisionTreeRegressor":""" Sets the value of :py:attr:`minInfoGain`. """returnself._set(minInfoGain=value)
[docs]@since("1.4.0")defsetMaxMemoryInMB(self,value:int)->"DecisionTreeRegressor":""" Sets the value of :py:attr:`maxMemoryInMB`. """returnself._set(maxMemoryInMB=value)
[docs]@since("1.4.0")defsetCacheNodeIds(self,value:bool)->"DecisionTreeRegressor":""" Sets the value of :py:attr:`cacheNodeIds`. """returnself._set(cacheNodeIds=value)
[docs]@since("1.4.0")defsetImpurity(self,value:str)->"DecisionTreeRegressor":""" Sets the value of :py:attr:`impurity`. """returnself._set(impurity=value)
[docs]@since("1.4.0")defsetCheckpointInterval(self,value:int)->"DecisionTreeRegressor":""" Sets the value of :py:attr:`checkpointInterval`. """returnself._set(checkpointInterval=value)
[docs]defsetSeed(self,value:int)->"DecisionTreeRegressor":""" Sets the value of :py:attr:`seed`. """returnself._set(seed=value)
[docs]@since("3.0.0")defsetWeightCol(self,value:str)->"DecisionTreeRegressor":""" Sets the value of :py:attr:`weightCol`. """returnself._set(weightCol=value)
[docs]@since("2.0.0")defsetVarianceCol(self,value:str)->"DecisionTreeRegressor":""" Sets the value of :py:attr:`varianceCol`. """returnself._set(varianceCol=value)
[docs]@inherit_docclassDecisionTreeRegressionModel(_JavaRegressionModel,_DecisionTreeModel,_DecisionTreeRegressorParams,JavaMLWritable,JavaMLReadable["DecisionTreeRegressionModel"],):""" Model fitted by :class:`DecisionTreeRegressor`. .. versionadded:: 1.4.0 """
[docs]@since("3.0.0")defsetVarianceCol(self,value:str)->"DecisionTreeRegressionModel":""" Sets the value of :py:attr:`varianceCol`. """returnself._set(varianceCol=value)
@propertydeffeatureImportances(self)->Vector:""" Estimate of the importance of each feature. This generalizes the idea of "Gini" importance to other losses, following the explanation of Gini importance from "Random Forests" documentation by Leo Breiman and Adele Cutler, and following the implementation from scikit-learn. This feature importance is calculated as follows: - importance(feature j) = sum (over nodes which split on feature j) of the gain, where gain is scaled by the number of instances passing through node - Normalize importances for tree to sum to 1. .. versionadded:: 2.0.0 Notes ----- Feature importance for single decision trees can have high variance due to correlated predictor variables. Consider using a :py:class:`RandomForestRegressor` to determine feature importance instead. """returnself._call_java("featureImportances")
class_RandomForestRegressorParams(_RandomForestParams,_TreeRegressorParams):""" Params for :py:class:`RandomForestRegressor` and :py:class:`RandomForestRegressionModel`. .. versionadded:: 3.0.0 """def__init__(self,*args:Any):super(_RandomForestRegressorParams,self).__init__(*args)self._setDefault(maxDepth=5,maxBins=32,minInstancesPerNode=1,minInfoGain=0.0,maxMemoryInMB=256,cacheNodeIds=False,checkpointInterval=10,impurity="variance",subsamplingRate=1.0,numTrees=20,featureSubsetStrategy="auto",leafCol="",minWeightFractionPerNode=0.0,bootstrap=True,)
[docs]defsetMaxDepth(self,value:int)->"RandomForestRegressor":""" Sets the value of :py:attr:`maxDepth`. """returnself._set(maxDepth=value)
[docs]defsetMaxBins(self,value:int)->"RandomForestRegressor":""" Sets the value of :py:attr:`maxBins`. """returnself._set(maxBins=value)
[docs]defsetMinInstancesPerNode(self,value:int)->"RandomForestRegressor":""" Sets the value of :py:attr:`minInstancesPerNode`. """returnself._set(minInstancesPerNode=value)
[docs]defsetMinInfoGain(self,value:float)->"RandomForestRegressor":""" Sets the value of :py:attr:`minInfoGain`. """returnself._set(minInfoGain=value)
[docs]defsetMaxMemoryInMB(self,value:int)->"RandomForestRegressor":""" Sets the value of :py:attr:`maxMemoryInMB`. """returnself._set(maxMemoryInMB=value)
[docs]defsetCacheNodeIds(self,value:bool)->"RandomForestRegressor":""" Sets the value of :py:attr:`cacheNodeIds`. """returnself._set(cacheNodeIds=value)
[docs]@since("1.4.0")defsetImpurity(self,value:str)->"RandomForestRegressor":""" Sets the value of :py:attr:`impurity`. """returnself._set(impurity=value)
[docs]@since("1.4.0")defsetNumTrees(self,value:int)->"RandomForestRegressor":""" Sets the value of :py:attr:`numTrees`. """returnself._set(numTrees=value)
[docs]@since("3.0.0")defsetBootstrap(self,value:bool)->"RandomForestRegressor":""" Sets the value of :py:attr:`bootstrap`. """returnself._set(bootstrap=value)
[docs]@since("1.4.0")defsetSubsamplingRate(self,value:float)->"RandomForestRegressor":""" Sets the value of :py:attr:`subsamplingRate`. """returnself._set(subsamplingRate=value)
[docs]@since("2.4.0")defsetFeatureSubsetStrategy(self,value:str)->"RandomForestRegressor":""" Sets the value of :py:attr:`featureSubsetStrategy`. """returnself._set(featureSubsetStrategy=value)
[docs]defsetCheckpointInterval(self,value:int)->"RandomForestRegressor":""" Sets the value of :py:attr:`checkpointInterval`. """returnself._set(checkpointInterval=value)
[docs]defsetSeed(self,value:int)->"RandomForestRegressor":""" Sets the value of :py:attr:`seed`. """returnself._set(seed=value)
[docs]@since("3.0.0")defsetWeightCol(self,value:str)->"RandomForestRegressor":""" Sets the value of :py:attr:`weightCol`. """returnself._set(weightCol=value)
[docs]@since("3.0.0")defsetMinWeightFractionPerNode(self,value:float)->"RandomForestRegressor":""" Sets the value of :py:attr:`minWeightFractionPerNode`. """returnself._set(minWeightFractionPerNode=value)
[docs]classRandomForestRegressionModel(_JavaRegressionModel[Vector],_TreeEnsembleModel,_RandomForestRegressorParams,JavaMLWritable,JavaMLReadable["RandomForestRegressionModel"],):""" Model fitted by :class:`RandomForestRegressor`. .. versionadded:: 1.4.0 """@property# type: ignore[misc]@since("2.0.0")deftrees(self)->List[DecisionTreeRegressionModel]:"""Trees in this ensemble. Warning: These have null parent Estimators."""return[DecisionTreeRegressionModel(m)forminlist(self._call_java("trees"))]@propertydeffeatureImportances(self)->Vector:""" Estimate of the importance of each feature. Each feature's importance is the average of its importance across all trees in the ensemble The importance vector is normalized to sum to 1. This method is suggested by Hastie et al. (Hastie, Tibshirani, Friedman. "The Elements of Statistical Learning, 2nd Edition." 2001.) and follows the implementation from scikit-learn. .. versionadded:: 2.0.0 Examples -------- DecisionTreeRegressionModel.featureImportances """returnself._call_java("featureImportances")
class_GBTRegressorParams(_GBTParams,_TreeRegressorParams):""" Params for :py:class:`GBTRegressor` and :py:class:`GBTRegressorModel`. .. versionadded:: 3.0.0 """supportedLossTypes:List[str]=["squared","absolute"]lossType:Param[str]=Param(Params._dummy(),"lossType","Loss function which GBT tries to minimize (case-insensitive). "+"Supported options: "+", ".join(supportedLossTypes),typeConverter=TypeConverters.toString,)def__init__(self,*args:Any):super(_GBTRegressorParams,self).__init__(*args)self._setDefault(maxDepth=5,maxBins=32,minInstancesPerNode=1,minInfoGain=0.0,maxMemoryInMB=256,cacheNodeIds=False,subsamplingRate=1.0,checkpointInterval=10,lossType="squared",maxIter=20,stepSize=0.1,impurity="variance",featureSubsetStrategy="all",validationTol=0.01,leafCol="",minWeightFractionPerNode=0.0,)@since("1.4.0")defgetLossType(self)->str:""" Gets the value of lossType or its default value. """returnself.getOrDefault(self.lossType)
[docs]@since("1.4.0")defsetMaxDepth(self,value:int)->"GBTRegressor":""" Sets the value of :py:attr:`maxDepth`. """returnself._set(maxDepth=value)
[docs]@since("1.4.0")defsetMaxBins(self,value:int)->"GBTRegressor":""" Sets the value of :py:attr:`maxBins`. """returnself._set(maxBins=value)
[docs]@since("1.4.0")defsetMinInstancesPerNode(self,value:int)->"GBTRegressor":""" Sets the value of :py:attr:`minInstancesPerNode`. """returnself._set(minInstancesPerNode=value)
[docs]@since("1.4.0")defsetMinInfoGain(self,value:float)->"GBTRegressor":""" Sets the value of :py:attr:`minInfoGain`. """returnself._set(minInfoGain=value)
[docs]@since("1.4.0")defsetMaxMemoryInMB(self,value:int)->"GBTRegressor":""" Sets the value of :py:attr:`maxMemoryInMB`. """returnself._set(maxMemoryInMB=value)
[docs]@since("1.4.0")defsetCacheNodeIds(self,value:bool)->"GBTRegressor":""" Sets the value of :py:attr:`cacheNodeIds`. """returnself._set(cacheNodeIds=value)
[docs]@since("1.4.0")defsetImpurity(self,value:str)->"GBTRegressor":""" Sets the value of :py:attr:`impurity`. """returnself._set(impurity=value)
[docs]@since("1.4.0")defsetLossType(self,value:str)->"GBTRegressor":""" Sets the value of :py:attr:`lossType`. """returnself._set(lossType=value)
[docs]@since("1.4.0")defsetSubsamplingRate(self,value:float)->"GBTRegressor":""" Sets the value of :py:attr:`subsamplingRate`. """returnself._set(subsamplingRate=value)
[docs]@since("2.4.0")defsetFeatureSubsetStrategy(self,value:str)->"GBTRegressor":""" Sets the value of :py:attr:`featureSubsetStrategy`. """returnself._set(featureSubsetStrategy=value)
[docs]@since("3.0.0")defsetValidationIndicatorCol(self,value:str)->"GBTRegressor":""" Sets the value of :py:attr:`validationIndicatorCol`. """returnself._set(validationIndicatorCol=value)
[docs]@since("1.4.0")defsetMaxIter(self,value:int)->"GBTRegressor":""" Sets the value of :py:attr:`maxIter`. """returnself._set(maxIter=value)
[docs]@since("1.4.0")defsetCheckpointInterval(self,value:int)->"GBTRegressor":""" Sets the value of :py:attr:`checkpointInterval`. """returnself._set(checkpointInterval=value)
[docs]@since("1.4.0")defsetSeed(self,value:int)->"GBTRegressor":""" Sets the value of :py:attr:`seed`. """returnself._set(seed=value)
[docs]@since("1.4.0")defsetStepSize(self,value:float)->"GBTRegressor":""" Sets the value of :py:attr:`stepSize`. """returnself._set(stepSize=value)
[docs]@since("3.0.0")defsetWeightCol(self,value:str)->"GBTRegressor":""" Sets the value of :py:attr:`weightCol`. """returnself._set(weightCol=value)
[docs]@since("3.0.0")defsetMinWeightFractionPerNode(self,value:float)->"GBTRegressor":""" Sets the value of :py:attr:`minWeightFractionPerNode`. """returnself._set(minWeightFractionPerNode=value)
[docs]classGBTRegressionModel(_JavaRegressionModel[Vector],_TreeEnsembleModel,_GBTRegressorParams,JavaMLWritable,JavaMLReadable["GBTRegressionModel"],):""" Model fitted by :class:`GBTRegressor`. .. versionadded:: 1.4.0 """@propertydeffeatureImportances(self)->Vector:""" Estimate of the importance of each feature. Each feature's importance is the average of its importance across all trees in the ensemble The importance vector is normalized to sum to 1. This method is suggested by Hastie et al. (Hastie, Tibshirani, Friedman. "The Elements of Statistical Learning, 2nd Edition." 2001.) and follows the implementation from scikit-learn. .. versionadded:: 2.0.0 Examples -------- DecisionTreeRegressionModel.featureImportances """returnself._call_java("featureImportances")@property# type: ignore[misc]@since("2.0.0")deftrees(self)->List[DecisionTreeRegressionModel]:"""Trees in this ensemble. Warning: These have null parent Estimators."""return[DecisionTreeRegressionModel(m)forminlist(self._call_java("trees"))]
[docs]defevaluateEachIteration(self,dataset:DataFrame,loss:str)->List[float]:""" Method to compute error or loss for every iteration of gradient boosting. .. versionadded:: 2.4.0 Parameters ---------- dataset : :py:class:`pyspark.sql.DataFrame` Test dataset to evaluate model on, where dataset is an instance of :py:class:`pyspark.sql.DataFrame` loss : str The loss function used to compute error. Supported options: squared, absolute """returnself._call_java("evaluateEachIteration",dataset,loss)
class_AFTSurvivalRegressionParams(_PredictorParams,HasMaxIter,HasTol,HasFitIntercept,HasAggregationDepth,HasMaxBlockSizeInMB):""" Params for :py:class:`AFTSurvivalRegression` and :py:class:`AFTSurvivalRegressionModel`. .. versionadded:: 3.0.0 """censorCol:Param[str]=Param(Params._dummy(),"censorCol","censor column name. The value of this column could be 0 or 1. "+"If the value is 1, it means the event has occurred i.e. "+"uncensored; otherwise censored.",typeConverter=TypeConverters.toString,)quantileProbabilities:Param[List[float]]=Param(Params._dummy(),"quantileProbabilities","quantile probabilities array. Values of the quantile probabilities array "+"should be in the range (0, 1) and the array should be non-empty.",typeConverter=TypeConverters.toListFloat,)quantilesCol:Param[str]=Param(Params._dummy(),"quantilesCol","quantiles column name. This column will output quantiles of "+"corresponding quantileProbabilities if it is set.",typeConverter=TypeConverters.toString,)def__init__(self,*args:Any):super(_AFTSurvivalRegressionParams,self).__init__(*args)self._setDefault(censorCol="censor",quantileProbabilities=[0.01,0.05,0.1,0.25,0.5,0.75,0.9,0.95,0.99],maxIter=100,tol=1e-6,maxBlockSizeInMB=0.0,)@since("1.6.0")defgetCensorCol(self)->str:""" Gets the value of censorCol or its default value. """returnself.getOrDefault(self.censorCol)@since("1.6.0")defgetQuantileProbabilities(self)->List[float]:""" Gets the value of quantileProbabilities or its default value. """returnself.getOrDefault(self.quantileProbabilities)@since("1.6.0")defgetQuantilesCol(self)->str:""" Gets the value of quantilesCol or its default value. """returnself.getOrDefault(self.quantilesCol)
[docs]@inherit_docclassAFTSurvivalRegression(_JavaRegressor["AFTSurvivalRegressionModel"],_AFTSurvivalRegressionParams,JavaMLWritable,JavaMLReadable["AFTSurvivalRegression"],):""" Accelerated Failure Time (AFT) Model Survival Regression Fit a parametric AFT survival regression model based on the Weibull distribution of the survival time. Notes ----- For more information see Wikipedia page on `AFT Model <https://en.wikipedia.org/wiki/Accelerated_failure_time_model>`_ Examples -------- >>> from pyspark.ml.linalg import Vectors >>> df = spark.createDataFrame([ ... (1.0, Vectors.dense(1.0), 1.0), ... (1e-40, Vectors.sparse(1, [], []), 0.0)], ["label", "features", "censor"]) >>> aftsr = AFTSurvivalRegression() >>> aftsr.setMaxIter(10) AFTSurvivalRegression... >>> aftsr.getMaxIter() 10 >>> aftsr.clear(aftsr.maxIter) >>> model = aftsr.fit(df) >>> model.getMaxBlockSizeInMB() 0.0 >>> model.setFeaturesCol("features") AFTSurvivalRegressionModel... >>> model.predict(Vectors.dense(6.3)) 1.0 >>> model.predictQuantiles(Vectors.dense(6.3)) DenseVector([0.0101, 0.0513, 0.1054, 0.2877, 0.6931, 1.3863, 2.3026, 2.9957, 4.6052]) >>> model.transform(df).show() +-------+---------+------+----------+ | label| features|censor|prediction| +-------+---------+------+----------+ | 1.0| [1.0]| 1.0| 1.0| |1.0E-40|(1,[],[])| 0.0| 1.0| +-------+---------+------+----------+ ... >>> aftsr_path = temp_path + "/aftsr" >>> aftsr.save(aftsr_path) >>> aftsr2 = AFTSurvivalRegression.load(aftsr_path) >>> aftsr2.getMaxIter() 100 >>> model_path = temp_path + "/aftsr_model" >>> model.save(model_path) >>> model2 = AFTSurvivalRegressionModel.load(model_path) >>> model.coefficients == model2.coefficients True >>> model.intercept == model2.intercept True >>> model.scale == model2.scale True >>> model.transform(df).take(1) == model2.transform(df).take(1) True .. versionadded:: 1.6.0 """_input_kwargs:Dict[str,Any]@keyword_onlydef__init__(self,*,featuresCol:str="features",labelCol:str="label",predictionCol:str="prediction",fitIntercept:bool=True,maxIter:int=100,tol:float=1e-6,censorCol:str="censor",quantileProbabilities:List[float]=[0.01,0.05,0.1,0.25,0.5,0.75,0.9,0.95,0.99,],# noqa: B005quantilesCol:Optional[str]=None,aggregationDepth:int=2,maxBlockSizeInMB:float=0.0,):""" __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \ fitIntercept=True, maxIter=100, tol=1E-6, censorCol="censor", \ quantileProbabilities=[0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99], \ quantilesCol=None, aggregationDepth=2, maxBlockSizeInMB=0.0) """super(AFTSurvivalRegression,self).__init__()self._java_obj=self._new_java_obj("org.apache.spark.ml.regression.AFTSurvivalRegression",self.uid)kwargs=self._input_kwargsself.setParams(**kwargs)
[docs]@since("1.6.0")defsetCensorCol(self,value:str)->"AFTSurvivalRegression":""" Sets the value of :py:attr:`censorCol`. """returnself._set(censorCol=value)
[docs]@since("1.6.0")defsetQuantileProbabilities(self,value:List[float])->"AFTSurvivalRegression":""" Sets the value of :py:attr:`quantileProbabilities`. """returnself._set(quantileProbabilities=value)
[docs]@since("1.6.0")defsetQuantilesCol(self,value:str)->"AFTSurvivalRegression":""" Sets the value of :py:attr:`quantilesCol`. """returnself._set(quantilesCol=value)
[docs]@since("1.6.0")defsetMaxIter(self,value:int)->"AFTSurvivalRegression":""" Sets the value of :py:attr:`maxIter`. """returnself._set(maxIter=value)
[docs]@since("1.6.0")defsetTol(self,value:float)->"AFTSurvivalRegression":""" Sets the value of :py:attr:`tol`. """returnself._set(tol=value)
[docs]@since("1.6.0")defsetFitIntercept(self,value:bool)->"AFTSurvivalRegression":""" Sets the value of :py:attr:`fitIntercept`. """returnself._set(fitIntercept=value)
[docs]@since("2.1.0")defsetAggregationDepth(self,value:int)->"AFTSurvivalRegression":""" Sets the value of :py:attr:`aggregationDepth`. """returnself._set(aggregationDepth=value)
[docs]@since("3.1.0")defsetMaxBlockSizeInMB(self,value:int)->"AFTSurvivalRegression":""" Sets the value of :py:attr:`maxBlockSizeInMB`. """returnself._set(maxBlockSizeInMB=value)
[docs]classAFTSurvivalRegressionModel(_JavaRegressionModel[Vector],_AFTSurvivalRegressionParams,JavaMLWritable,JavaMLReadable["AFTSurvivalRegressionModel"],):""" Model fitted by :class:`AFTSurvivalRegression`. .. versionadded:: 1.6.0 """
[docs]@since("3.0.0")defsetQuantileProbabilities(self,value:List[float])->"AFTSurvivalRegressionModel":""" Sets the value of :py:attr:`quantileProbabilities`. """returnself._set(quantileProbabilities=value)
[docs]@since("3.0.0")defsetQuantilesCol(self,value:str)->"AFTSurvivalRegressionModel":""" Sets the value of :py:attr:`quantilesCol`. """returnself._set(quantilesCol=value)
@property# type: ignore[misc]@since("2.0.0")defcoefficients(self)->Vector:""" Model coefficients. """returnself._call_java("coefficients")@property# type: ignore[misc]@since("1.6.0")defintercept(self)->float:""" Model intercept. """returnself._call_java("intercept")@property# type: ignore[misc]@since("1.6.0")defscale(self)->float:""" Model scale parameter. """returnself._call_java("scale")
class_GeneralizedLinearRegressionParams(_PredictorParams,HasFitIntercept,HasMaxIter,HasTol,HasRegParam,HasWeightCol,HasSolver,HasAggregationDepth,):""" Params for :py:class:`GeneralizedLinearRegression` and :py:class:`GeneralizedLinearRegressionModel`. .. versionadded:: 3.0.0 """family:Param[str]=Param(Params._dummy(),"family","The name of family which is a description of "+"the error distribution to be used in the model. Supported options: "+"gaussian (default), binomial, poisson, gamma and tweedie.",typeConverter=TypeConverters.toString,)link:Param[str]=Param(Params._dummy(),"link","The name of link function which provides the "+"relationship between the linear predictor and the mean of the distribution "+"function. Supported options: identity, log, inverse, logit, probit, cloglog "+"and sqrt.",typeConverter=TypeConverters.toString,)linkPredictionCol:Param[str]=Param(Params._dummy(),"linkPredictionCol","link prediction (linear "+"predictor) column name",typeConverter=TypeConverters.toString,)variancePower:Param[float]=Param(Params._dummy(),"variancePower","The power in the variance function "+"of the Tweedie distribution which characterizes the relationship "+"between the variance and mean of the distribution. Only applicable "+"for the Tweedie family. Supported values: 0 and [1, Inf).",typeConverter=TypeConverters.toFloat,)linkPower:Param[float]=Param(Params._dummy(),"linkPower","The index in the power link function. "+"Only applicable to the Tweedie family.",typeConverter=TypeConverters.toFloat,)solver:Param[str]=Param(Params._dummy(),"solver","The solver algorithm for optimization. Supported "+"options: irls.",typeConverter=TypeConverters.toString,)offsetCol:Param[str]=Param(Params._dummy(),"offsetCol","The offset column name. If this is not set "+"or empty, we treat all instance offsets as 0.0",typeConverter=TypeConverters.toString,)def__init__(self,*args:Any):super(_GeneralizedLinearRegressionParams,self).__init__(*args)self._setDefault(family="gaussian",maxIter=25,tol=1e-6,regParam=0.0,solver="irls",variancePower=0.0,aggregationDepth=2,)@since("2.0.0")defgetFamily(self)->str:""" Gets the value of family or its default value. """returnself.getOrDefault(self.family)@since("2.0.0")defgetLinkPredictionCol(self)->str:""" Gets the value of linkPredictionCol or its default value. """returnself.getOrDefault(self.linkPredictionCol)@since("2.0.0")defgetLink(self)->str:""" Gets the value of link or its default value. """returnself.getOrDefault(self.link)@since("2.2.0")defgetVariancePower(self)->float:""" Gets the value of variancePower or its default value. """returnself.getOrDefault(self.variancePower)@since("2.2.0")defgetLinkPower(self)->float:""" Gets the value of linkPower or its default value. """returnself.getOrDefault(self.linkPower)@since("2.3.0")defgetOffsetCol(self)->str:""" Gets the value of offsetCol or its default value. """returnself.getOrDefault(self.offsetCol)
[docs]@inherit_docclassGeneralizedLinearRegression(_JavaRegressor["GeneralizedLinearRegressionModel"],_GeneralizedLinearRegressionParams,JavaMLWritable,JavaMLReadable["GeneralizedLinearRegression"],):""" Generalized Linear Regression. Fit a Generalized Linear Model specified by giving a symbolic description of the linear predictor (link function) and a description of the error distribution (family). It supports "gaussian", "binomial", "poisson", "gamma" and "tweedie" as family. Valid link functions for each family is listed below. The first link function of each family is the default one. * "gaussian" -> "identity", "log", "inverse" * "binomial" -> "logit", "probit", "cloglog" * "poisson" -> "log", "identity", "sqrt" * "gamma" -> "inverse", "identity", "log" * "tweedie" -> power link function specified through "linkPower". \ The default link power in the tweedie family is 1 - variancePower. .. versionadded:: 2.0.0 Notes ----- For more information see Wikipedia page on `GLM <https://en.wikipedia.org/wiki/Generalized_linear_model>`_ Examples -------- >>> from pyspark.ml.linalg import Vectors >>> df = spark.createDataFrame([ ... (1.0, Vectors.dense(0.0, 0.0)), ... (1.0, Vectors.dense(1.0, 2.0)), ... (2.0, Vectors.dense(0.0, 0.0)), ... (2.0, Vectors.dense(1.0, 1.0)),], ["label", "features"]) >>> glr = GeneralizedLinearRegression(family="gaussian", link="identity", linkPredictionCol="p") >>> glr.setRegParam(0.1) GeneralizedLinearRegression... >>> glr.getRegParam() 0.1 >>> glr.clear(glr.regParam) >>> glr.setMaxIter(10) GeneralizedLinearRegression... >>> glr.getMaxIter() 10 >>> glr.clear(glr.maxIter) >>> model = glr.fit(df) >>> model.setFeaturesCol("features") GeneralizedLinearRegressionModel... >>> model.getMaxIter() 25 >>> model.getAggregationDepth() 2 >>> transformed = model.transform(df) >>> abs(transformed.head().prediction - 1.5) < 0.001 True >>> abs(transformed.head().p - 1.5) < 0.001 True >>> model.coefficients DenseVector([1.5..., -1.0...]) >>> model.numFeatures 2 >>> abs(model.intercept - 1.5) < 0.001 True >>> glr_path = temp_path + "/glr" >>> glr.save(glr_path) >>> glr2 = GeneralizedLinearRegression.load(glr_path) >>> glr.getFamily() == glr2.getFamily() True >>> model_path = temp_path + "/glr_model" >>> model.save(model_path) >>> model2 = GeneralizedLinearRegressionModel.load(model_path) >>> model.intercept == model2.intercept True >>> model.coefficients[0] == model2.coefficients[0] True >>> model.transform(df).take(1) == model2.transform(df).take(1) True """_input_kwargs:Dict[str,Any]@keyword_onlydef__init__(self,*,labelCol:str="label",featuresCol:str="features",predictionCol:str="prediction",family:str="gaussian",link:Optional[str]=None,fitIntercept:bool=True,maxIter:int=25,tol:float=1e-6,regParam:float=0.0,weightCol:Optional[str]=None,solver:str="irls",linkPredictionCol:Optional[str]=None,variancePower:float=0.0,linkPower:Optional[float]=None,offsetCol:Optional[str]=None,aggregationDepth:int=2,):""" __init__(self, \\*, labelCol="label", featuresCol="features", predictionCol="prediction", \ family="gaussian", link=None, fitIntercept=True, maxIter=25, tol=1e-6, \ regParam=0.0, weightCol=None, solver="irls", linkPredictionCol=None, \ variancePower=0.0, linkPower=None, offsetCol=None, aggregationDepth=2) """super(GeneralizedLinearRegression,self).__init__()self._java_obj=self._new_java_obj("org.apache.spark.ml.regression.GeneralizedLinearRegression",self.uid)kwargs=self._input_kwargsself.setParams(**kwargs)
[docs]@since("2.0.0")defsetFamily(self,value:str)->"GeneralizedLinearRegression":""" Sets the value of :py:attr:`family`. """returnself._set(family=value)
[docs]@since("2.0.0")defsetLinkPredictionCol(self,value:str)->"GeneralizedLinearRegression":""" Sets the value of :py:attr:`linkPredictionCol`. """returnself._set(linkPredictionCol=value)
[docs]@since("2.0.0")defsetLink(self,value:str)->"GeneralizedLinearRegression":""" Sets the value of :py:attr:`link`. """returnself._set(link=value)
[docs]@since("2.2.0")defsetVariancePower(self,value:float)->"GeneralizedLinearRegression":""" Sets the value of :py:attr:`variancePower`. """returnself._set(variancePower=value)
[docs]@since("2.2.0")defsetLinkPower(self,value:float)->"GeneralizedLinearRegression":""" Sets the value of :py:attr:`linkPower`. """returnself._set(linkPower=value)
[docs]@since("2.3.0")defsetOffsetCol(self,value:str)->"GeneralizedLinearRegression":""" Sets the value of :py:attr:`offsetCol`. """returnself._set(offsetCol=value)
[docs]@since("2.0.0")defsetMaxIter(self,value:int)->"GeneralizedLinearRegression":""" Sets the value of :py:attr:`maxIter`. """returnself._set(maxIter=value)
[docs]@since("2.0.0")defsetRegParam(self,value:float)->"GeneralizedLinearRegression":""" Sets the value of :py:attr:`regParam`. """returnself._set(regParam=value)
[docs]@since("2.0.0")defsetTol(self,value:float)->"GeneralizedLinearRegression":""" Sets the value of :py:attr:`tol`. """returnself._set(tol=value)
[docs]@since("2.0.0")defsetFitIntercept(self,value:bool)->"GeneralizedLinearRegression":""" Sets the value of :py:attr:`fitIntercept`. """returnself._set(fitIntercept=value)
[docs]@since("2.0.0")defsetWeightCol(self,value:str)->"GeneralizedLinearRegression":""" Sets the value of :py:attr:`weightCol`. """returnself._set(weightCol=value)
[docs]@since("2.0.0")defsetSolver(self,value:str)->"GeneralizedLinearRegression":""" Sets the value of :py:attr:`solver`. """returnself._set(solver=value)
[docs]@since("3.0.0")defsetAggregationDepth(self,value:int)->"GeneralizedLinearRegression":""" Sets the value of :py:attr:`aggregationDepth`. """returnself._set(aggregationDepth=value)
[docs]classGeneralizedLinearRegressionModel(_JavaRegressionModel[Vector],_GeneralizedLinearRegressionParams,JavaMLWritable,JavaMLReadable["GeneralizedLinearRegressionModel"],HasTrainingSummary["GeneralizedLinearRegressionTrainingSummary"],):""" Model fitted by :class:`GeneralizedLinearRegression`. .. versionadded:: 2.0.0 """
[docs]@since("3.0.0")defsetLinkPredictionCol(self,value:str)->"GeneralizedLinearRegressionModel":""" Sets the value of :py:attr:`linkPredictionCol`. """returnself._set(linkPredictionCol=value)
@property# type: ignore[misc]@since("2.0.0")defcoefficients(self)->Vector:""" Model coefficients. """returnself._call_java("coefficients")@property# type: ignore[misc]@since("2.0.0")defintercept(self)->float:""" Model intercept. """returnself._call_java("intercept")@property# type: ignore[misc]@since("2.0.0")defsummary(self)->"GeneralizedLinearRegressionTrainingSummary":""" Gets summary (residuals, deviance, p-values) of model on training set. An exception is thrown if `trainingSummary is None`. """ifself.hasSummary:returnGeneralizedLinearRegressionTrainingSummary(super(GeneralizedLinearRegressionModel,self).summary)else:raiseRuntimeError("No training summary available for this %s"%self.__class__.__name__)
[docs]defevaluate(self,dataset:DataFrame)->"GeneralizedLinearRegressionSummary":""" Evaluates the model on a test dataset. .. versionadded:: 2.0.0 Parameters ---------- dataset : :py:class:`pyspark.sql.DataFrame` Test dataset to evaluate model on, where dataset is an instance of :py:class:`pyspark.sql.DataFrame` """ifnotisinstance(dataset,DataFrame):raiseTypeError("dataset must be a DataFrame but got %s."%type(dataset))java_glr_summary=self._call_java("evaluate",dataset)returnGeneralizedLinearRegressionSummary(java_glr_summary)
[docs]classGeneralizedLinearRegressionSummary(JavaWrapper):""" Generalized linear regression results evaluated on a dataset. .. versionadded:: 2.0.0 """@property# type: ignore[misc]@since("2.0.0")defpredictions(self)->DataFrame:""" Predictions output by the model's `transform` method. """returnself._call_java("predictions")@property# type: ignore[misc]@since("2.0.0")defpredictionCol(self)->str:""" Field in :py:attr:`predictions` which gives the predicted value of each instance. This is set to a new column name if the original model's `predictionCol` is not set. """returnself._call_java("predictionCol")@property# type: ignore[misc]@since("2.2.0")defnumInstances(self)->int:""" Number of instances in DataFrame predictions. """returnself._call_java("numInstances")@property# type: ignore[misc]@since("2.0.0")defrank(self)->int:""" The numeric rank of the fitted linear model. """returnself._call_java("rank")@property# type: ignore[misc]@since("2.0.0")defdegreesOfFreedom(self)->int:""" Degrees of freedom. """returnself._call_java("degreesOfFreedom")@property# type: ignore[misc]@since("2.0.0")defresidualDegreeOfFreedom(self)->int:""" The residual degrees of freedom. """returnself._call_java("residualDegreeOfFreedom")@property# type: ignore[misc]@since("2.0.0")defresidualDegreeOfFreedomNull(self)->int:""" The residual degrees of freedom for the null model. """returnself._call_java("residualDegreeOfFreedomNull")
[docs]defresiduals(self,residualsType:str="deviance")->DataFrame:""" Get the residuals of the fitted model by type. .. versionadded:: 2.0.0 Parameters ---------- residualsType : str, optional The type of residuals which should be returned. Supported options: deviance (default), pearson, working, and response. """returnself._call_java("residuals",residualsType)
@property# type: ignore[misc]@since("2.0.0")defnullDeviance(self)->float:""" The deviance for the null model. """returnself._call_java("nullDeviance")@property# type: ignore[misc]@since("2.0.0")defdeviance(self)->float:""" The deviance for the fitted model. """returnself._call_java("deviance")@property# type: ignore[misc]@since("2.0.0")defdispersion(self)->float:""" The dispersion of the fitted model. It is taken as 1.0 for the "binomial" and "poisson" families, and otherwise estimated by the residual Pearson's Chi-Squared statistic (which is defined as sum of the squares of the Pearson residuals) divided by the residual degrees of freedom. """returnself._call_java("dispersion")@property# type: ignore[misc]@since("2.0.0")defaic(self)->float:""" Akaike's "An Information Criterion"(AIC) for the fitted model. """returnself._call_java("aic")
[docs]@inherit_docclassGeneralizedLinearRegressionTrainingSummary(GeneralizedLinearRegressionSummary):""" Generalized linear regression training results. .. versionadded:: 2.0.0 """@property# type: ignore[misc]@since("2.0.0")defnumIterations(self)->int:""" Number of training iterations. """returnself._call_java("numIterations")@property# type: ignore[misc]@since("2.0.0")defsolver(self)->str:""" The numeric solver used for training. """returnself._call_java("solver")@property# type: ignore[misc]@since("2.0.0")defcoefficientStandardErrors(self)->List[float]:""" Standard error of estimated coefficients and intercept. If :py:attr:`GeneralizedLinearRegression.fitIntercept` is set to True, then the last element returned corresponds to the intercept. """returnself._call_java("coefficientStandardErrors")@property# type: ignore[misc]@since("2.0.0")deftValues(self)->List[float]:""" T-statistic of estimated coefficients and intercept. If :py:attr:`GeneralizedLinearRegression.fitIntercept` is set to True, then the last element returned corresponds to the intercept. """returnself._call_java("tValues")@property# type: ignore[misc]@since("2.0.0")defpValues(self)->List[float]:""" Two-sided p-value of estimated coefficients and intercept. If :py:attr:`GeneralizedLinearRegression.fitIntercept` is set to True, then the last element returned corresponds to the intercept. """returnself._call_java("pValues")def__repr__(self)->str:returnself._call_java("toString")
class_FactorizationMachinesParams(_PredictorParams,HasMaxIter,HasStepSize,HasTol,HasSolver,HasSeed,HasFitIntercept,HasRegParam,HasWeightCol,):""" Params for :py:class:`FMRegressor`, :py:class:`FMRegressionModel`, :py:class:`FMClassifier` and :py:class:`FMClassifierModel`. .. versionadded:: 3.0.0 """factorSize:Param[int]=Param(Params._dummy(),"factorSize","Dimensionality of the factor vectors, "+"which are used to get pairwise interactions between variables",typeConverter=TypeConverters.toInt,)fitLinear:Param[bool]=Param(Params._dummy(),"fitLinear","whether to fit linear term (aka 1-way term)",typeConverter=TypeConverters.toBoolean,)miniBatchFraction:Param[float]=Param(Params._dummy(),"miniBatchFraction","fraction of the input data "+"set that should be used for one iteration of gradient descent",typeConverter=TypeConverters.toFloat,)initStd:Param[float]=Param(Params._dummy(),"initStd","standard deviation of initial coefficients",typeConverter=TypeConverters.toFloat,)solver=Param(Params._dummy(),"solver","The solver algorithm for optimization. Supported "+"options: gd, adamW. (Default adamW)",typeConverter=TypeConverters.toString,)def__init__(self,*args:Any):super(_FactorizationMachinesParams,self).__init__(*args)self._setDefault(factorSize=8,fitIntercept=True,fitLinear=True,regParam=0.0,miniBatchFraction=1.0,initStd=0.01,maxIter=100,stepSize=1.0,tol=1e-6,solver="adamW",)@since("3.0.0")defgetFactorSize(self)->int:""" Gets the value of factorSize or its default value. """returnself.getOrDefault(self.factorSize)@since("3.0.0")defgetFitLinear(self)->bool:""" Gets the value of fitLinear or its default value. """returnself.getOrDefault(self.fitLinear)@since("3.0.0")defgetMiniBatchFraction(self)->float:""" Gets the value of miniBatchFraction or its default value. """returnself.getOrDefault(self.miniBatchFraction)@since("3.0.0")defgetInitStd(self)->float:""" Gets the value of initStd or its default value. """returnself.getOrDefault(self.initStd)
[docs]@since("3.0.0")defsetFactorSize(self,value:int)->"FMRegressor":""" Sets the value of :py:attr:`factorSize`. """returnself._set(factorSize=value)
[docs]@since("3.0.0")defsetFitLinear(self,value:bool)->"FMRegressor":""" Sets the value of :py:attr:`fitLinear`. """returnself._set(fitLinear=value)
[docs]@since("3.0.0")defsetMiniBatchFraction(self,value:float)->"FMRegressor":""" Sets the value of :py:attr:`miniBatchFraction`. """returnself._set(miniBatchFraction=value)
[docs]@since("3.0.0")defsetInitStd(self,value:float)->"FMRegressor":""" Sets the value of :py:attr:`initStd`. """returnself._set(initStd=value)
[docs]@since("3.0.0")defsetMaxIter(self,value:int)->"FMRegressor":""" Sets the value of :py:attr:`maxIter`. """returnself._set(maxIter=value)
[docs]@since("3.0.0")defsetStepSize(self,value:float)->"FMRegressor":""" Sets the value of :py:attr:`stepSize`. """returnself._set(stepSize=value)
[docs]@since("3.0.0")defsetTol(self,value:float)->"FMRegressor":""" Sets the value of :py:attr:`tol`. """returnself._set(tol=value)
[docs]@since("3.0.0")defsetSolver(self,value:str)->"FMRegressor":""" Sets the value of :py:attr:`solver`. """returnself._set(solver=value)
[docs]@since("3.0.0")defsetSeed(self,value:int)->"FMRegressor":""" Sets the value of :py:attr:`seed`. """returnself._set(seed=value)
[docs]@since("3.0.0")defsetFitIntercept(self,value:bool)->"FMRegressor":""" Sets the value of :py:attr:`fitIntercept`. """returnself._set(fitIntercept=value)
[docs]@since("3.0.0")defsetRegParam(self,value:float)->"FMRegressor":""" Sets the value of :py:attr:`regParam`. """returnself._set(regParam=value)
[docs]classFMRegressionModel(_JavaRegressionModel,_FactorizationMachinesParams,JavaMLWritable,JavaMLReadable["FMRegressionModel"],):""" Model fitted by :class:`FMRegressor`. .. versionadded:: 3.0.0 """@property# type: ignore[misc]@since("3.0.0")defintercept(self)->float:""" Model intercept. """returnself._call_java("intercept")@property# type: ignore[misc]@since("3.0.0")deflinear(self)->Vector:""" Model linear term. """returnself._call_java("linear")@property# type: ignore[misc]@since("3.0.0")deffactors(self)->Matrix:""" Model factor term. """returnself._call_java("factors")
if__name__=="__main__":importdoctestimportpyspark.ml.regressionfrompyspark.sqlimportSparkSessionglobs=pyspark.ml.regression.__dict__.copy()# The small batch size here ensures that we see multiple batches,# even in these small test examples:spark=SparkSession.builder.master("local[2]").appName("ml.regression tests").getOrCreate()sc=spark.sparkContextglobs["sc"]=scglobs["spark"]=sparkimporttempfiletemp_path=tempfile.mkdtemp()globs["temp_path"]=temp_pathtry:(failure_count,test_count)=doctest.testmod(globs=globs,optionflags=doctest.ELLIPSIS)spark.stop()finally:fromshutilimportrmtreetry:rmtree(temp_path)exceptOSError:passiffailure_count:sys.exit(-1)