Spark 4.0.0 ScalaDoc - org.apache.spark.rdd.RDD (original) (raw)

abstract class RDD[T] extends Serializable with Logging

 

Ordering

1. Alphabetic
2. By Inheritance

Inherited

1. RDD

2. Logging

3. Serializable

4. AnyRef

5. Any

6. Hide All

7. Show All

Visibility

1. Public
2. Protected

Instance Constructors

1.  new RDD(oneParent: RDD[_])(implicit arg0: ClassTag[T])
2.  new RDD(_sc: SparkContext, deps: Seq[Dependency[_]])(implicit arg0: ClassTag[T])

Type Members

1.  implicit class LogStringContext extends AnyRef

Concrete Value Members

1.  final def !=(arg0: Any): Boolean
2.  final def ##: Int
3.  def ++(other: RDD[T]): RDD[T]
4.  final def ==(arg0: Any): Boolean
5.  def aggregate[U](zeroValue: U)(seqOp: (U, T) => U, combOp: (U, U) => U)(implicit arg0: ClassTag[U]): U
6.  final def asInstanceOf[T0]: T0
7.  def barrier(): RDDBarrier[T]
8.  def cache(): RDD.this.type
9.  def cartesian[U](other: RDD[U])(implicit arg0: ClassTag[U]): RDD[(T, U)]
10.  def checkpoint(): Unit
11.  def cleanShuffleDependencies(blocking: Boolean = false): Unit
12.  def clearDependencies(): Unit
13.  def clone(): AnyRef
14.  def coalesce(numPartitions: Int, shuffle: Boolean = false, partitionCoalescer: Option[PartitionCoalescer] = Option.empty)(implicit ord: Ordering[T] = null): RDD[T]
15.  def collect[U](f: PartialFunction[T, U])(implicit arg0: ClassTag[U]): RDD[U]
16.  def collect(): Array[T]
17.  def context: SparkContext
18.  def count(): Long
19.  def countApprox(timeout: Long, confidence: Double = 0.95): PartialResult[BoundedDouble]
20.  def countApproxDistinct(relativeSD: Double = 0.05): Long
21.  def countApproxDistinct(p: Int, sp: Int): Long
22.  def countByValue()(implicit ord: Ordering[T] = null): Map[T, Long]
23.  def countByValueApprox(timeout: Long, confidence: Double = 0.95)(implicit ord: Ordering[T] = null): PartialResult[Map[T, BoundedDouble]]
24.  final def dependencies: Seq[Dependency[_]]
25.  def distinct(): RDD[T]
26.  def distinct(numPartitions: Int)(implicit ord: Ordering[T] = null): RDD[T]
27.  final def eq(arg0: AnyRef): Boolean
28.  def equals(arg0: AnyRef): Boolean
29.  def filter(f: (T) => Boolean): RDD[T]
30.  def first(): T
31.  def firstParent[U](implicit arg0: ClassTag[U]): RDD[U]
32.  def flatMap[U](f: (T) => IterableOnce[U])(implicit arg0: ClassTag[U]): RDD[U]
33.  def fold(zeroValue: T)(op: (T, T) => T): T
34.  def foreach(f: (T) => Unit): Unit
35.  def foreachPartition(f: (Iterator[T]) => Unit): Unit
36.  def getCheckpointFile: Option[String]
37.  final def getClass(): Class[_ <: AnyRef]
38.  def getDependencies: Seq[Dependency[_]]
39.  final def getNumPartitions: Int
40.  def getOutputDeterministicLevel: DeterministicLevel.Value
41.  def getPreferredLocations(split: Partition): Seq[String]
42.  def getResourceProfile(): ResourceProfile
43.  def getStorageLevel: StorageLevel
44.  def glom(): RDD[Array[T]]
45.  def groupBy[K](f: (T) => K, p: Partitioner)(implicit kt: ClassTag[K], ord: Ordering[K] = null): RDD[(K, Iterable[T])]
46.  def groupBy[K](f: (T) => K, numPartitions: Int)(implicit kt: ClassTag[K]): RDD[(K, Iterable[T])]
47.  def groupBy[K](f: (T) => K)(implicit kt: ClassTag[K]): RDD[(K, Iterable[T])]
48.  def hashCode(): Int
49.  val id: Int
50.  def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
51.  def initializeLogIfNecessary(isInterpreter: Boolean): Unit
52.  def intersection(other: RDD[T], numPartitions: Int): RDD[T]
53.  def intersection(other: RDD[T], partitioner: Partitioner)(implicit ord: Ordering[T] = null): RDD[T]
54.  def intersection(other: RDD[T]): RDD[T]
55.  lazy val isBarrier_: Boolean
56.  def isCheckpointed: Boolean
57.  def isEmpty(): Boolean
58.  final def isInstanceOf[T0]: Boolean
59.  def isTraceEnabled(): Boolean
60.  final def iterator(split: Partition, context: TaskContext): Iterator[T]
61.  def keyBy[K](f: (T) => K): RDD[(K, T)]
62.  def localCheckpoint(): RDD.this.type
63.  def log: Logger
64.  def logDebug(msg: => String, throwable: Throwable): Unit
65.  def logDebug(entry: LogEntry, throwable: Throwable): Unit
66.  def logDebug(entry: LogEntry): Unit
67.  def logDebug(msg: => String): Unit
68.  def logError(msg: => String, throwable: Throwable): Unit
69.  def logError(entry: LogEntry, throwable: Throwable): Unit
70.  def logError(entry: LogEntry): Unit
71.  def logError(msg: => String): Unit
72.  def logInfo(msg: => String, throwable: Throwable): Unit
73.  def logInfo(entry: LogEntry, throwable: Throwable): Unit
74.  def logInfo(entry: LogEntry): Unit
75.  def logInfo(msg: => String): Unit
76.  def logName: String
77.  def logTrace(msg: => String, throwable: Throwable): Unit
78.  def logTrace(entry: LogEntry, throwable: Throwable): Unit
79.  def logTrace(entry: LogEntry): Unit
80.  def logTrace(msg: => String): Unit
81.  def logWarning(msg: => String, throwable: Throwable): Unit
82.  def logWarning(entry: LogEntry, throwable: Throwable): Unit
83.  def logWarning(entry: LogEntry): Unit
84.  def logWarning(msg: => String): Unit
85.  def map[U](f: (T) => U)(implicit arg0: ClassTag[U]): RDD[U]
86.  def mapPartitions[U](f: (Iterator[T]) => Iterator[U], preservesPartitioning: Boolean = false)(implicit arg0: ClassTag[U]): RDD[U]
87.  def mapPartitionsWithEvaluator[U](evaluatorFactory: PartitionEvaluatorFactory[T, U])(implicit arg0: ClassTag[U]): RDD[U]
88.  def mapPartitionsWithIndex[U](f: (Int, Iterator[T]) => Iterator[U], preservesPartitioning: Boolean = false)(implicit arg0: ClassTag[U]): RDD[U]
89.  def max()(implicit ord: Ordering[T]): T
90.  def min()(implicit ord: Ordering[T]): T
91.  var name: String
92.  final def ne(arg0: AnyRef): Boolean
93.  final def notify(): Unit
94.  final def notifyAll(): Unit
95.  def parent[U](j: Int)(implicit arg0: ClassTag[U]): RDD[U]
96.  val partitioner: Option[Partitioner]
97.  final def partitions: Array[Partition]
98.  def persist(): RDD.this.type
99.  def persist(newLevel: StorageLevel): RDD.this.type
100.  def pipe(command: Seq[String], env: Map[String, String] = Map(), printPipeContext: ((String) => Unit) => Unit = null, printRDDElement: (T, (String) => Unit) => Unit = null, separateWorkingDir: Boolean = false, bufferSize: Int = 8192, encoding: String = Codec.defaultCharsetCodec.name): RDD[String]
101.  def pipe(command: String, env: Map[String, String]): RDD[String]
102.  def pipe(command: String): RDD[String]
103.  final def preferredLocations(split: Partition): Seq[String]
104.  def randomSplit(weights: Array[Double], seed: Long = Utils.random.nextLong): Array[RDD[T]]
105.  def reduce(f: (T, T) => T): T
106.  def repartition(numPartitions: Int)(implicit ord: Ordering[T] = null): RDD[T]
107.  def sample(withReplacement: Boolean, fraction: Double, seed: Long = Utils.random.nextLong): RDD[T]
108.  def saveAsObjectFile(path: String): Unit
109.  def saveAsTextFile(path: String, codec: Class[_ <: CompressionCodec]): Unit
110.  def saveAsTextFile(path: String): Unit
111.  def setName(_name: String): RDD.this.type
112.  def sortBy[K](f: (T) => K, ascending: Boolean = true, numPartitions: Int = this.partitions.length)(implicit ord: Ordering[K], ctag: ClassTag[K]): RDD[T]
113.  def sparkContext: SparkContext
114.  def subtract(other: RDD[T], p: Partitioner)(implicit ord: Ordering[T] = null): RDD[T]
115.  def subtract(other: RDD[T], numPartitions: Int): RDD[T]
116.  def subtract(other: RDD[T]): RDD[T]
117.  final def synchronized[T0](arg0: => T0): T0
118.  def take(num: Int): Array[T]
119.  def takeOrdered(num: Int)(implicit ord: Ordering[T]): Array[T]
120.  def takeSample(withReplacement: Boolean, num: Int, seed: Long = Utils.random.nextLong): Array[T]
121.  def toDebugString: String
122.  def toJavaRDD(): JavaRDD[T]
123.  def toLocalIterator: Iterator[T]
124.  def toString(): String
125.  def top(num: Int)(implicit ord: Ordering[T]): Array[T]
126.  def treeAggregate[U](zeroValue: U, seqOp: (U, T) => U, combOp: (U, U) => U, depth: Int, finalAggregateOnExecutor: Boolean)(implicit arg0: ClassTag[U]): U
127.  def treeAggregate[U](zeroValue: U)(seqOp: (U, T) => U, combOp: (U, U) => U, depth: Int = 2)(implicit arg0: ClassTag[U]): U
128.  def treeReduce(f: (T, T) => T, depth: Int = 2): T
129.  def union(other: RDD[T]): RDD[T]
130.  def unpersist(blocking: Boolean = false): RDD.this.type
131.  final def wait(arg0: Long, arg1: Int): Unit
132.  final def wait(arg0: Long): Unit
133.  final def wait(): Unit
134.  def withLogContext(context: Map[String, String])(body: => Unit): Unit
135.  def withResources(rp: ResourceProfile): RDD.this.type
136.  def zip[U](other: RDD[U])(implicit arg0: ClassTag[U]): RDD[(T, U)]
137.  def zipPartitions[B, C, D, V](rdd2: RDD[B], rdd3: RDD[C], rdd4: RDD[D])(f: (Iterator[T], Iterator[B], Iterator[C], Iterator[D]) => Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[C], arg2: ClassTag[D], arg3: ClassTag[V]): RDD[V]
138.  def zipPartitions[B, C, D, V](rdd2: RDD[B], rdd3: RDD[C], rdd4: RDD[D], preservesPartitioning: Boolean)(f: (Iterator[T], Iterator[B], Iterator[C], Iterator[D]) => Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[C], arg2: ClassTag[D], arg3: ClassTag[V]): RDD[V]
139.  def zipPartitions[B, C, V](rdd2: RDD[B], rdd3: RDD[C])(f: (Iterator[T], Iterator[B], Iterator[C]) => Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[C], arg2: ClassTag[V]): RDD[V]
140.  def zipPartitions[B, C, V](rdd2: RDD[B], rdd3: RDD[C], preservesPartitioning: Boolean)(f: (Iterator[T], Iterator[B], Iterator[C]) => Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[C], arg2: ClassTag[V]): RDD[V]
141.  def zipPartitions[B, V](rdd2: RDD[B])(f: (Iterator[T], Iterator[B]) => Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[V]): RDD[V]
142.  def zipPartitions[B, V](rdd2: RDD[B], preservesPartitioning: Boolean)(f: (Iterator[T], Iterator[B]) => Iterator[V])(implicit arg0: ClassTag[B], arg1: ClassTag[V]): RDD[V]
143.  def zipPartitionsWithEvaluator[U](rdd2: RDD[T], evaluatorFactory: PartitionEvaluatorFactory[T, U])(implicit arg0: ClassTag[U]): RDD[U]
144.  def zipWithIndex(): RDD[(T, Long)]
145.  def zipWithUniqueId(): RDD[(T, Long)]

Deprecated Value Members

1.  def finalize(): Unit

Inherited from Logging

Inherited from Serializable

Inherited from AnyRef

Inherited from Any

Ungrouped