Flink Window基本概念與實現原理

/** 每3個事件，計算最近4個事件訊息 */
keyedStream.countWindow(4,3);
複製程式碼

/** 依據WindowAssigner例項化WindowedStream */
public <W extends Window> WindowedStream<T, KEY, W> window(WindowAssigner<? super T, W> assigner) {
    return new WindowedStream<>(this, assigner);
}複製程式碼

/** WindowedStream構造器 */
public WindowedStream(KeyedStream<T, K> input, WindowAssigner<? super T, W> windowAssigner) {
    this.input = input;
    this.windowAssigner = windowAssigner;
    this.trigger = windowAssigner.getDefaultTrigger(input.getExecutionEnvironment());
}
複製程式碼

public WindowedStream<T, K, W> trigger(Trigger<? super T, ? super W> trigger) {
    if (windowAssigner instanceof MergingWindowAssigner && !trigger.canMerge()) {
        throw new UnsupportedOperationException();
    }
    if (windowAssigner instanceof BaseAlignedWindowAssigner) {
        throw new UnsupportedOperationException();
    }
    this.trigger = trigger;
    return this;
}複製程式碼

public WindowedStream<T, K, W> evictor(Evictor<? super T, ? super W> evictor) {
    if (windowAssigner instanceof BaseAlignedWindowAssigner) {
        throw new UnsupportedOperationException();
    }
    this.evictor = evictor;
    return this;
}複製程式碼

public class EvictingWindowOperator extends WindowOperator {
    private final Evictor evictor;
}複製程式碼

private void emitWindowContents(W window, Iterable<StreamRecord<IN>> contents, ListState<StreamRecord<IN>> windowState) throws Exception {
    /** Window處理前資料清理 */
    evictorContext.evictBefore(recordsWithTimestamp, Iterables.size(recordsWithTimestamp));
    /** Window處理 */
    userFunction.process(triggerContext.key, triggerContext.window, processContext, projectedContents, timestampedCollector);
    /** Window處理後資料清理 */
    evictorContext.evictAfter(recordsWithTimestamp, Iterables.size(recordsWithTimestamp));
}
複製程式碼

/** 滾動計數視窗 */
public WindowedStream<T, KEY, GlobalWindow> countWindow(long size) {
    return window(GlobalWindows.create()).trigger(PurgingTrigger.of(CountTrigger.of(size)));
}

/** 滑動計數視窗 */
public WindowedStream<T, KEY, GlobalWindow> countWindow(long size, long slide) {
    return window(GlobalWindows.create())
        .evictor(CountEvictor.of(size))
        .trigger(CountTrigger.of(slide));
}
複製程式碼

/** GlobalWindows是一個WindowAssigner實現,這裡只展示實現assignWindows的程式碼片段 */
public class GlobalWindows extends WindowAssigner<Object, GlobalWindow> {
    /** 返回一個GlobalWindow */
    public Collection<GlobalWindow> assignWindows(Object element, long timestamp, WindowAssignerContext context) {
        return Collections.singletonList(GlobalWindow.get());
    }
}
複製程式碼

/** GlobalWindow是一個Window */
public class GlobalWindow extends Window {
    private static final GlobalWindow INSTANCE = new GlobalWindow();
    /** 永遠返回GlobalWindow單例 */
    public static GlobalWindow get() {
        return INSTANCE;
    }
}
複製程式碼

/** PurgingTrigger代理的Trigger */
private Trigger<T, W> nestedTrigger;
/** PurgingTrigger私有構造器 */
private PurgingTrigger(Trigger<T, W> nestedTrigger) {
    this.nestedTrigger = nestedTrigger;
}

/** 為代理的Trigger構造一個PurgingTrigger例項 */
public static <T, W extends Window> PurgingTrigger<T, W> of(Trigger<T, W> nestedTrigger) {
    return new PurgingTrigger<>(nestedTrigger);
}
複製程式碼

/** CountTrigger實現 */
public TriggerResult onElement(Object element, long timestamp, W window, TriggerContext ctx) throws Exception {
    ReducingState<Long> count = ctx.getPartitionedState(stateDesc);
    count.add(1L);
    if (count.get() >= maxCount) {
        count.clear();
        return TriggerResult.FIRE;
    }
    return TriggerResult.CONTINUE;
}
/** PurgingTrigger實現 */
public TriggerResult onElement(T element, long timestamp, W window, TriggerContext ctx) throws Exception {
    TriggerResult triggerResult = nestedTrigger.onElement(element, timestamp, window, ctx);
    return triggerResult.isFire() ? TriggerResult.FIRE_AND_PURGE : triggerResult;
}
複製程式碼

public static TumblingProcessingTimeWindows of(Time size) {
    return new TumblingProcessingTimeWindows(size.toMilliseconds(), 0);
}
public static TumblingProcessingTimeWindows of(Time size, Time offset) {
    return new TumblingProcessingTimeWindows(size.toMilliseconds(), offset.toMilliseconds());
}
複製程式碼

/** TumblingProcessingTimeWindows構造器 */
private TumblingProcessingTimeWindows(long size, long offset) {
    if (offset < 0 || offset >= size) {
        throw new IllegalArgumentException();
    }
    this.size = size;
    this.offset = offset;
}
複製程式碼

public Collection<TimeWindow> assignWindows(Object element, long timestamp, WindowAssignerContext context) {
    final long now = context.getCurrentProcessingTime();
    long start = TimeWindow.getWindowStartWithOffset(now, offset, size);
    return Collections.singletonList(new TimeWindow(start, start + size));
}
複製程式碼

Window[2019年1月1日 12:00:00, 2019年1月1日 12:00:10)複製程式碼

Window[2019年1月1日 12:10:00, 2019年1月1日 12:10:10)複製程式碼

keyedStream.window(TumblingProcessingTimeWindows.of(Time.minutes(1), Time.seconds(15)))複製程式碼

Window[2019年1月1日 11:59:15, 2019年1月1日 12:00:15)複製程式碼

Window[2019年1月1日 12:00:15, 2019年1月1日 12:01:15)複製程式碼

public boolean equals(Object o) {
    if (this == o) {
        return true;
    }
    if (o == null || getClass() != o.getClass()) {
        return false;
    }
    TimeWindow window = (TimeWindow) o;
    return end == window.end && start == window.start;
}
複製程式碼

ScheduledThreadPoolExecutor.schedule(new TriggerTask(), timeEndTime - systemTime, TimeUnit.MILLISECONDS);
複製程式碼