def segment(self, document):
assert(len(document) > <NUM_LIT:0> and len([d for d in document if not isinstance(d, str)]) == <NUM_LIT:0>)<EOL>if len(document) < <NUM_LIT:3>:<EOL><INDENT>return [<NUM_LIT:1>] + [<NUM_LIT:0> for _ in range(len(document) - <NUM_LIT:1>)]<EOL><DEDENT>n = len(document)<EOL>self.window = min(self.window, n)<EOL>cnts = [Counter(self.tokenizer.tokenize(document[i])) for i in range(n)]<EOL>self.sim = np.zeros((n, n))<EOL>for i in range(n):<EOL><INDENT>for j in range(i, n):<EOL><INDENT>self.sim[i][j] = cosine_sim(cnts[i], cnts[j])<EOL>self.sim[j][i] = self.sim[i][j]<EOL><DEDENT><DEDENT>self.rank = np.zeros((n, n))<EOL>for i in range(n):<EOL><INDENT>for j in range(i, n):<EOL><INDENT>r1 = max(<NUM_LIT:0>, i - self.window + <NUM_LIT:1>)<EOL>r2 = min(n - <NUM_LIT:1>, i + self.window - <NUM_LIT:1>)<EOL>c1 = max(<NUM_LIT:0>, j - self.window + <NUM_LIT:1>)<EOL>c2 = min(n - <NUM_LIT:1>, j + self.window - <NUM_LIT:1>)<EOL>sublist = self.sim[r1:(r2 + <NUM_LIT:1>), c1:(c2+<NUM_LIT:1>)].flatten()<EOL>lowlist = [x for x in sublist if x < self.sim[i][j]]<EOL>self.rank[i][j] = <NUM_LIT:1.0> * len(lowlist) / ((r2 - r1 + <NUM_LIT:1>) * (c2 - c1 + <NUM_LIT:1>))<EOL>self.rank[j][i] = self.rank[i][j]<EOL><DEDENT><DEDENT>self.sm = np.zeros((n, n))<EOL>prefix_sm = np.zeros((n, n))<EOL>for i in range(n):<EOL><INDENT>for j in range(n):<EOL><INDENT>prefix_sm[i][j] = self.rank[i][j]<EOL>if i - <NUM_LIT:1> >= <NUM_LIT:0>: prefix_sm[i][j] += prefix_sm[i - <NUM_LIT:1>][j]<EOL>if j - <NUM_LIT:1> >= <NUM_LIT:0>: prefix_sm[i][j] += prefix_sm[i][j - <NUM_LIT:1>]<EOL>if i - <NUM_LIT:1> >= <NUM_LIT:0> and j - <NUM_LIT:1> >= <NUM_LIT:0>: prefix_sm[i][j] -= prefix_sm[i - <NUM_LIT:1>][j - <NUM_LIT:1>]<EOL><DEDENT><DEDENT>for i in range(n):<EOL><INDENT>for j in range(i, n):<EOL><INDENT>if i == <NUM_LIT:0>:<EOL><INDENT>self.sm[i][j] = prefix_sm[j][j]<EOL><DEDENT>else:<EOL><INDENT>self.sm[i][j] = prefix_sm[j][j] - prefix_sm[i - <NUM_LIT:1>][j]- prefix_sm[j][i - <NUM_LIT:1>] + prefix_sm[i - <NUM_LIT:1>][i - <NUM_LIT:1>]<EOL><DEDENT>self.sm[j][i] = self.sm[i][j]<EOL><DEDENT><DEDENT>D = <NUM_LIT:1.0> * self.sm[<NUM_LIT:0>][n - <NUM_LIT:1>] / (n * n)<EOL>darr, region_arr, idx = [D], [Region(<NUM_LIT:0>, n - <NUM_LIT:1>, self.sm)], []<EOL>sum_region, sum_area = float(self.sm[<NUM_LIT:0>][n - <NUM_LIT:1>]), float(n * n)<EOL>for i in range(n - <NUM_LIT:1>):<EOL><INDENT>mx, pos = -<NUM_LIT>, -<NUM_LIT:1><EOL>for j, region in enumerate(region_arr):<EOL><INDENT>if region.l == region.r:<EOL><INDENT>continue<EOL><DEDENT>region.split(self.sm)<EOL>den = sum_area - region.area + region.lch.area + region.rch.area<EOL>cur = (sum_region - region.tot + region.lch.tot + region.rch.tot) / den<EOL>if cur > mx:<EOL><INDENT>mx, pos = cur, j<EOL><DEDENT><DEDENT>assert(pos >= <NUM_LIT:0>)<EOL>tmp = region_arr[pos]<EOL>region_arr[pos] = tmp.rch<EOL>region_arr.insert(pos, tmp.lch)<EOL>sum_region += tmp.lch.tot + tmp.rch.tot - tmp.tot<EOL>sum_area += tmp.lch.area + tmp.rch.area - tmp.area<EOL>darr.append(sum_region / sum_area)<EOL>idx.append(tmp.best_pos)<EOL><DEDENT>dgrad = [(darr[i + <NUM_LIT:1>] - darr[i]) for i in range(len(darr) - <NUM_LIT:1>)]<EOL>smooth_dgrad = [dgrad[i] for i in range(len(dgrad))]<EOL>if len(dgrad) > <NUM_LIT:1>:<EOL><INDENT>smooth_dgrad[<NUM_LIT:0>] = (dgrad[<NUM_LIT:0>] * <NUM_LIT:2> + dgrad[<NUM_LIT:1>]) / <NUM_LIT><EOL>smooth_dgrad[-<NUM_LIT:1>] = (dgrad[-<NUM_LIT:1>] * <NUM_LIT:2> + dgrad[-<NUM_LIT:2>]) / <NUM_LIT><EOL><DEDENT>for i in range(<NUM_LIT:1>, len(dgrad) - <NUM_LIT:1>):<EOL><INDENT>smooth_dgrad[i] = (dgrad[i - <NUM_LIT:1>] + <NUM_LIT:2> * dgrad[i] + dgrad[i + <NUM_LIT:1>]) / <NUM_LIT><EOL><DEDENT>dgrad = smooth_dgrad<EOL>avg, stdev = np.average(dgrad), np.std(dgrad)<EOL>cutoff = avg + self.std_coeff * stdev<EOL>assert(len(idx) == len(dgrad))<EOL>above_cutoff_idx = [i for i in range(len(dgrad)) if dgrad[i] >= cutoff]<EOL>if len(above_cutoff_idx) == <NUM_LIT:0>: boundary = []<EOL>else: boundary = idx[:max(above_cutoff_idx) + <NUM_LIT:1>]<EOL>ret = [<NUM_LIT:0> for _ in range(n)]<EOL>for i in boundary:<EOL><INDENT>ret[i] = <NUM_LIT:1><EOL>for j in range(i - <NUM_LIT:1>, i + <NUM_LIT:2>):<EOL><INDENT>if j >= <NUM_LIT:0> and j < n and j != i and ret[j] == <NUM_LIT:1>:<EOL><INDENT>ret[i] = <NUM_LIT:0><EOL>break<EOL><DEDENT><DEDENT><DEDENT>return [<NUM_LIT:1>] + ret[:-<NUM_LIT:1>]<EOL>
document: list[str]
return list[int],
    i-th element denotes whether exists a boundary right before paragraph i(0 indexed)