Sys path append(os path join(os path dirname

import sys
import os.path
import cv2

sys.path.append(os.path.join(os.path.dirname(__file__), '..'))

from manim_imports_ext import *

import warnings

warnings.warn("""
Warning: This file makes use of
ContinualAnimation, which has since
been deprecated
""")
from nn.network import *

#force_skipping

#revert_to_original_skipping_status
DEFAULT_GAUSS_BLUR_CONFIG = {
"ksize" : (5, 5),
"sigmaX" : 10,
"sigmaY" : 10,
}

DEFAULT_CANNY_CONFIG = {

"threshold1" : 100,
"threshold2" : 200,
}
def get_edges(image_array):
blurred = cv2.GaussianBlur(
image_array,
**DEFAULT_GAUSS_BLUR_CONFIG
)
edges = cv2.Canny(
blurred,
**DEFAULT_CANNY_CONFIG
)
return edges

class WrappedImage(Group):

CONFIG = {
"rect_kwargs" : {
"color" : BLUE,
"buff" : SMALL_BUFF,
}
}
def __init__(self, image_mobject, **kwargs):
Group.__init__(self, **kwargs)
rect = SurroundingRectangle(
image_mobject, **self.rect_kwargs
)
self.add(rect, image_mobject)

class PixelsAsSquares(VGroup):

CONFIG = {
"height" : 2,
}
def __init__(self, image_mobject, **kwargs):
VGroup.__init__(self, **kwargs)
for row in image_mobject.pixel_array:
for rgba in row:
square = Square(
stroke_width = 0,
fill_opacity = rgba[3]/255.0,
fill_color = rgba_to_color(rgba/255.0),
)
self.add(square)
self.arrange_in_grid(
*image_mobject.pixel_array.shape[:2],
buff = 0
)
self.replace(image_mobject)

class PixelsFromVect(PixelsAsSquares):

def __init__(self, vect, **kwargs):
PixelsAsSquares.__init__(self,
ImageMobject(layer_to_image_array(vect)),
**kwargs
)

class MNistMobject(WrappedImage):

def __init__(self, vect, **kwargs):
WrappedImage.__init__(self,
ImageMobject(layer_to_image_array(vect)),
**kwargs
)

class NetworkMobject(VGroup):

CONFIG = {
"neuron_radius" : 0.15,
"neuron_to_neuron_buff" : MED_SMALL_BUFF,
"layer_to_layer_buff" : LARGE_BUFF,
"neuron_stroke_color" : BLUE,
"neuron_stroke_width" : 3,
"neuron_fill_color" : GREEN,
"edge_color" : GREY_B,
"edge_stroke_width" : 2,
"edge_propogation_color" : YELLOW,
"edge_propogation_time" : 1,
"max_shown_neurons" : 16,
"brace_for_large_layers" : True,
"average_shown_activation_of_large_layer" : True,
"include_output_labels" : False,
}
def __init__(self, neural_network, **kwargs):
VGroup.__init__(self, **kwargs)
self.neural_network = neural_network
self.layer_sizes = neural_network.sizes
self.add_neurons()
self.add_edges()

def add_neurons(self):

layers = VGroup(*[
self.get_layer(size)
for size in self.layer_sizes
])
layers.arrange(RIGHT, buff = self.layer_to_layer_buff)
self.layers = layers
self.add(self.layers)
if self.include_output_labels:
self.add_output_labels()

def get_layer(self, size):

layer = VGroup()
n_neurons = size
if n_neurons > self.max_shown_neurons:
n_neurons = self.max_shown_neurons
neurons = VGroup(*[
Circle(
radius = self.neuron_radius,
stroke_color = self.neuron_stroke_color,
stroke_width = self.neuron_stroke_width,
fill_color = self.neuron_fill_color,
fill_opacity = 0,
)
for x in range(n_neurons)
])
neurons.arrange(
DOWN, buff = self.neuron_to_neuron_buff
)
for neuron in neurons:
neuron.edges_in = VGroup()
neuron.edges_out = VGroup()
layer.neurons = neurons
layer.add(neurons)

if size > n_neurons:

dots = Tex("\\vdots")
dots.move_to(neurons)
VGroup(*neurons[:len(neurons) // 2]).next_to(
dots, UP, MED_SMALL_BUFF
)
VGroup(*neurons[len(neurons) // 2:]).next_to(
dots, DOWN, MED_SMALL_BUFF
)
layer.dots = dots
layer.add(dots)
if self.brace_for_large_layers:
brace = Brace(layer, LEFT)
brace_label = brace.get_tex(str(size))
layer.brace = brace
layer.brace_label = brace_label
layer.add(brace, brace_label)

return layer

def add_edges(self):

self.edge_groups = VGroup()
for l1, l2 in zip(self.layers[:-1], self.layers[1:]):
edge_group = VGroup()
for n1, n2 in it.product(l1.neurons, l2.neurons):
edge = self.get_edge(n1, n2)
edge_group.add(edge)
n1.edges_out.add(edge)
n2.edges_in.add(edge)
self.edge_groups.add(edge_group)
self.add_to_back(self.edge_groups)

def get_edge(self, neuron1, neuron2):

return Line(
neuron1.get_center(),
neuron2.get_center(),
buff = self.neuron_radius,
stroke_color = self.edge_color,
stroke_width = self.edge_stroke_width,
)

def get_active_layer(self, layer_index, activation_vector):

layer = self.layers[layer_index].deepcopy()
self.activate_layer(layer, activation_vector)
return layer

def activate_layer(self, layer, activation_vector):

n_neurons = len(layer.neurons)
av = activation_vector
def arr_to_num(arr):
return (np.sum(arr > 0.1) / float(len(arr)))**(1./3)

if len(av) > n_neurons:

if self.average_shown_activation_of_large_layer:
indices = np.arange(n_neurons)
indices *= int(len(av)/n_neurons)
indices = list(indices)
indices.append(len(av))
av = np.array([
arr_to_num(av[i1:i2])
for i1, i2 in zip(indices[:-1], indices[1:])
])
else:
av = np.append(
av[:n_neurons/2],
av[-n_neurons/2:],
)
for activation, neuron in zip(av, layer.neurons):
neuron.set_fill(
color = self.neuron_fill_color,
opacity = activation
)
return layer

def activate_layers(self, input_vector):

activations = self.neural_network.get_activation_of_all_layers(input_vector)
for activation, layer in zip(activations, self.layers):
self.activate_layer(layer, activation)

def deactivate_layers(self):

all_neurons = VGroup(*it.chain(*[
layer.neurons
for layer in self.layers
]))
all_neurons.set_fill(opacity = 0)
return self

def get_edge_propogation_animations(self, index):

edge_group_copy = self.edge_groups[index].copy()
edge_group_copy.set_stroke(
self.edge_propogation_color,
width = 1.5*self.edge_stroke_width
)
return [ShowCreationThenDestruction(
edge_group_copy,
run_time = self.edge_propogation_time,
lag_ratio = 0.5
)]

def add_output_labels(self):

self.output_labels = VGroup()
for n, neuron in enumerate(self.layers[-1].neurons):
label = Tex(str(n))
label.set_height(0.75*neuron.get_height())
label.move_to(neuron)
label.shift(neuron.get_width()*RIGHT)
self.output_labels.add(label)
self.add(self.output_labels)

class MNistNetworkMobject(NetworkMobject):

CONFIG = {
"neuron_to_neuron_buff" : SMALL_BUFF,
"layer_to_layer_buff" : 1.5,
"edge_stroke_width" : 1,
"include_output_labels" : True,
}

def __init__(self, **kwargs):

network = get_pretrained_network()
NetworkMobject.__init__(self, network, **kwargs)

class NetworkScene(Scene):

CONFIG = {
"layer_sizes" : [8, 6, 6, 4],
"network_mob_config" : {},
}
def setup(self):
self.add_network()

def add_network(self):

self.network = Network(sizes = self.layer_sizes)
self.network_mob = NetworkMobject(
self.network,
**self.network_mob_config
)
self.add(self.network_mob)

def feed_forward(self, input_vector, false_confidence = False, added_anims = None):

if added_anims is None:
added_anims = []
activations = self.network.get_activation_of_all_layers(
input_vector
)
if false_confidence:
i = np.argmax(activations[-1])
activations[-1] *= 0
activations[-1][i] = 1.0
for i, activation in enumerate(activations):
self.show_activation_of_layer(i, activation, added_anims)
added_anims = []

def show_activation_of_layer(self, layer_index, activation_vector, added_anims = None):

if added_anims is None:
added_anims = []
layer = self.network_mob.layers[layer_index]
active_layer = self.network_mob.get_active_layer(
layer_index, activation_vector
)
anims = [Transform(layer, active_layer)]
if layer_index > 0:
anims += self.network_mob.get_edge_propogation_animations(
layer_index-1
)
anims += added_anims
self.play(*anims)

def remove_random_edges(self, prop = 0.9):

for edge_group in self.network_mob.edge_groups:
for edge in list(edge_group):
if np.random.random() < prop:
edge_group.remove(edge)

def make_transparent(image_mob):

alpha_vect = np.array(
image_mob.pixel_array[:,:,0],
dtype = 'uint8'
)
image_mob.set_color(WHITE)
image_mob.pixel_array[:,:,3] = alpha_vect
return image_mob

###############################

class ExampleThrees(PiCreatureScene):

def construct(self):
self.show_initial_three()
self.show_alternate_threes()
self.resolve_remaining_threes()
self.show_alternate_digits()

def show_initial_three(self):

randy = self.pi_creature

self.three_mobs = self.get_three_mobs()

three_mob = self.three_mobs[0]
three_mob_copy = three_mob[1].copy()
three_mob_copy.sort(lambda p : np.dot(p, DOWN+RIGHT))

braces = VGroup(*[Brace(three_mob, v) for v in (LEFT, UP)])

brace_labels = VGroup(*[
brace.get_text("28px")
for brace in braces
])

bubble = randy.get_bubble(height = 4, width = 6)

three_mob.generate_target()
three_mob.target.set_height(1)
three_mob.target.next_to(bubble[-1].get_left(), RIGHT, LARGE_BUFF)
arrow = Arrow(LEFT, RIGHT, color = BLUE)
arrow.next_to(three_mob.target, RIGHT)
real_three = Tex("3")
real_three.set_height(0.8)
real_three.next_to(arrow, RIGHT)

self.play(

FadeIn(three_mob[0]),
LaggedStartMap(FadeIn, three_mob[1])
)
self.wait()
self.play(
LaggedStartMap(
DrawBorderThenFill, three_mob_copy,
run_time = 3,
stroke_color = WHITE,
remover = True,
),
randy.change, "sassy",
*it.chain(
list(map(GrowFromCenter, braces)),
list(map(FadeIn, brace_labels))
)
)
self.wait()
self.play(
ShowCreation(bubble),
MoveToTarget(three_mob),
FadeOut(braces),
FadeOut(brace_labels),
randy.change, "pondering"
)
self.play(
ShowCreation(arrow),
Write(real_three)
)
self.wait()

self.bubble = bubble

self.arrow = arrow
self.real_three = real_three

def show_alternate_threes(self):

randy = self.pi_creature

three = self.three_mobs[0]

three.generate_target()
three.target[0].set_fill(opacity = 0, family = False)
for square in three.target[1]:
yellow_rgb = color_to_rgb(YELLOW)
square_rgb = color_to_rgb(square.get_fill_color())
square.set_fill(
rgba_to_color(yellow_rgb*square_rgb),
opacity = 0.5
)

alt_threes = VGroup(*self.three_mobs[1:])

alt_threes.arrange(DOWN)
alt_threes.set_height(FRAME_HEIGHT - 2)
alt_threes.to_edge(RIGHT)

for alt_three in alt_threes:

self.add(alt_three)
self.wait(0.5)
self.play(
randy.change, "plain",
*list(map(FadeOut, [
self.bubble, self.arrow, self.real_three
])) + [MoveToTarget(three)]
)
for alt_three in alt_threes[:2]:
self.play(three.replace, alt_three)
self.wait()
for moving_three in three, alt_threes[1]:
moving_three.generate_target()
moving_three.target.next_to(alt_threes, LEFT, LARGE_BUFF)
moving_three.target[0].set_stroke(width = 0)
moving_three.target[1].space_out_submobjects(1.5)
self.play(MoveToTarget(
moving_three, lag_ratio = 0.5
))
self.play(
Animation(randy),
moving_three.replace, randy.eyes[1],
moving_three.scale, 0.7,
run_time = 2,
lag_ratio = 0.5,
)
self.play(
Animation(randy),
FadeOut(moving_three)
)

self.remaining_threes = [alt_threes[0], alt_threes[2]]

def resolve_remaining_threes(self):

randy = self.pi_creature

left_three, right_three = self.remaining_threes

equals = Tex("=")
equals.move_to(self.arrow)
for three, vect in (left_three, LEFT), (right_three, RIGHT):
three.generate_target()
three.target.set_height(1)
three.target.next_to(equals, vect)

self.play(

randy.change, "thinking",
ShowCreation(self.bubble),
MoveToTarget(left_three),
MoveToTarget(right_three),
Write(equals),
)
self.wait()

self.equals = equals

def show_alternate_digits(self):

randy = self.pi_creature
cross = Cross(self.equals)
cross.stretch_to_fit_height(0.5)
three = self.remaining_threes[1]

image_map = get_organized_images()

arrays = [image_map[k][0] for k in range(8, 4, -1)]
alt_mobs = [
WrappedImage(
PixelsAsSquares(ImageMobject(layer_to_image_array(arr))),
color = GREY_B,
buff = 0
).replace(three)
for arr in arrays
]

self.play(

randy.change, "sassy",
Transform(three, alt_mobs[0]),
ShowCreation(cross)
)
self.wait()
for mob in alt_mobs[1:]:
self.play(Transform(three, mob))
self.wait()

######

def create_pi_creature(self):
return Randolph().to_corner(DOWN+LEFT)

def get_three_mobs(self):

three_arrays = get_organized_images()[3][:4]
three_mobs = VGroup()
for three_array in three_arrays:
im_mob = ImageMobject(
layer_to_image_array(three_array),
height = 4,
)
pixel_mob = PixelsAsSquares(im_mob)
three_mob = WrappedImage(
pixel_mob,
color = GREY_B,
buff = 0
)
three_mobs.add(three_mob)
return three_mobs

class BrainAndHow(Scene):

def construct(self):
brain = SVGMobject(file_name = "brain")
brain.set_height(2)
brain.set_fill(GREY_B)
brain_outline = brain.copy()
brain_outline.set_fill(opacity = 0)
brain_outline.set_stroke(BLUE_B, 3)

how = TexText("How?!?")

how.scale(2)
how.next_to(brain, UP)

self.add(brain)

self.play(Write(how))
for x in range(2):
self.play(
ShowPassingFlash(
brain_outline,
time_width = 0.5,
run_time = 2
)
)
self.wait()

class WriteAProgram(Scene):

def construct(self):
three_array = get_organized_images()[3][0]
im_mob = ImageMobject(layer_to_image_array(three_array))
three = PixelsAsSquares(im_mob)
three.sort(lambda p : np.dot(p, DOWN+RIGHT))
three.set_height(6)
three.next_to(ORIGIN, LEFT)
three_rect = SurroundingRectangle(
three,
color = BLUE,
buff = SMALL_BUFF
)

numbers = VGroup()

for square in three:
rgb = square.fill_rgb
num = DecimalNumber(
square.fill_rgb[0],
num_decimal_places = 1
)
num.set_stroke(width = 1)
color = rgba_to_color(1 - (rgb + 0.2)/1.2)
num.set_color(color)
num.set_width(0.7*square.get_width())
num.move_to(square)
numbers.add(num)

arrow = Arrow(LEFT, RIGHT, color = BLUE)

arrow.next_to(three, RIGHT)

choices = VGroup(*[Tex(str(n)) for n in range(10)])

choices.arrange(DOWN)
choices.set_height(FRAME_HEIGHT - 1)
choices.next_to(arrow, RIGHT)

self.play(

LaggedStartMap(DrawBorderThenFill, three),
ShowCreation(three_rect)
)
self.play(Write(numbers))
self.play(
ShowCreation(arrow),
LaggedStartMap(FadeIn, choices),
)

rect = SurroundingRectangle(choices[0], buff = SMALL_BUFF)

q_mark = Tex("?")
q_mark.next_to(rect, RIGHT)
self.play(ShowCreation(rect))
for n in 8, 1, 5, 3:
self.play(
rect.move_to, choices[n],
MaintainPositionRelativeTo(q_mark, rect)
)
self.wait(1)
choice = choices[3]
choices.remove(choice)
choice.add(rect)
self.play(
choice.scale, 1.5,
choice.next_to, arrow, RIGHT,
FadeOut(choices),
FadeOut(q_mark),
)
self.wait(2)

class LayOutPlan(TeacherStudentsScene, NetworkScene):

def setup(self):
TeacherStudentsScene.setup(self)
NetworkScene.setup(self)
self.remove(self.network_mob)

def construct(self):

self.force_skipping()

self.show_words()

self.show_network()
self.show_math()
self.ask_about_layers()
self.show_learning()
self.show_videos()

def show_words(self):

words = VGroup(
TexText("Machine", "learning").set_color(GREEN),
TexText("Neural network").set_color(BLUE),
)
words.next_to(self.teacher.get_corner(UP+LEFT), UP)
words[0].save_state()
words[0].shift(DOWN)
words[0].fade(1)

self.play(

words[0].restore,
self.teacher.change, "raise_right_hand",
self.get_student_changes("pondering", "erm", "sassy")
)
self.play(
words[0].shift, MED_LARGE_BUFF*UP,
FadeIn(words[1]),
)
self.change_student_modes(
*["pondering"]*3,
look_at_arg = words
)
self.play(words.to_corner, UP+RIGHT)

self.words = words

def show_network(self):

network_mob = self.network_mob
network_mob.next_to(self.students, UP)

self.play(

ReplacementTransform(
VGroup(self.words[1].copy()),
network_mob.layers
),
self.get_student_changes(
*["confused"]*3,
lag_ratio = 0
),
self.teacher.change, "plain",
run_time = 1
)
self.play(ShowCreation(
network_mob.edge_groups,
lag_ratio = 0.5,
run_time = 2,
rate_func=linear,
))
in_vect = np.random.random(self.network.sizes[0])
self.feed_forward(in_vect)

def show_math(self):

equation = Tex(
"\\textbf{a}_{l+1}", "=",
"\\sigma(",
"W_l", "\\textbf{a}_l", "+", "b_l",
")"
)
equation.set_color_by_tex_to_color_map({
"\\textbf{a}" : GREEN,
})
equation.move_to(self.network_mob.get_corner(UP+RIGHT))
equation.to_edge(UP)

self.play(Write(equation, run_time = 2))

self.wait()

self.equation = equation

def ask_about_layers(self):

self.student_says(
"Why the layers?",
student_index = 2,
bubble_kwargs = {"direction" : LEFT}
)
self.wait()
self.play(RemovePiCreatureBubble(self.students[2]))

def show_learning(self):

word = self.words[0][1].copy()
rect = SurroundingRectangle(word, color = YELLOW)
self.network_mob.neuron_fill_color = YELLOW

layer = self.network_mob.layers[-1]

activation = np.zeros(len(layer.neurons))
activation[1] = 1.0
active_layer = self.network_mob.get_active_layer(
-1, activation
)
word_group = VGroup(word, rect)
word_group.generate_target()
word_group.target.move_to(self.equation, LEFT)
word_group.target[0].set_color(YELLOW)
word_group.target[1].set_stroke(width = 0)

self.play(ShowCreation(rect))

self.play(
Transform(layer, active_layer),
FadeOut(self.equation),
MoveToTarget(word_group),
)
for edge_group in reversed(self.network_mob.edge_groups):
edge_group.generate_target()
for edge in edge_group.target:
edge.set_stroke(
YELLOW,
width = 4*np.random.random()**2
)
self.play(MoveToTarget(edge_group))
self.wait()

self.learning_word = word

def show_videos(self):

network_mob = self.network_mob
learning = self.learning_word
structure = TexText("Structure")
structure.set_color(YELLOW)
videos = VGroup(*[
VideoIcon().set_fill(RED)
for x in range(2)
])
videos.set_height(1.5)
videos.arrange(RIGHT, buff = LARGE_BUFF)
videos.next_to(self.students, UP, LARGE_BUFF)

network_mob.generate_target()

network_mob.target.set_height(0.8*videos[0].get_height())
network_mob.target.move_to(videos[0])
learning.generate_target()
learning.target.next_to(videos[1], UP)
structure.next_to(videos[0], UP)
structure.shift(0.5*SMALL_BUFF*UP)

self.revert_to_original_skipping_status()

self.play(
MoveToTarget(network_mob),
MoveToTarget(learning)
)
self.play(
DrawBorderThenFill(videos[0]),
FadeIn(structure),
self.get_student_changes(*["pondering"]*3)
)
self.wait()
self.play(DrawBorderThenFill(videos[1]))
self.wait()

class PreviewMNistNetwork(NetworkScene):

CONFIG = {
"n_examples" : 15,
"network_mob_config" : {},
}
def construct(self):
self.remove_random_edges(0.7) #Remove?

training_data, validation_data, test_data = load_data_wrapper()

for data in test_data[:self.n_examples]:
self.feed_in_image(data[0])

def feed_in_image(self, in_vect):

image = PixelsFromVect(in_vect)
image.next_to(self.network_mob, LEFT, LARGE_BUFF, UP)
image.shift_onto_screen()
image_rect = SurroundingRectangle(image, color = BLUE)
start_neurons = self.network_mob.layers[0].neurons.copy()
start_neurons.set_stroke(WHITE, width = 0)
start_neurons.set_fill(WHITE, 0)

self.play(FadeIn(image), FadeIn(image_rect))

self.feed_forward(in_vect, added_anims = [
self.get_image_to_layer_one_animation(image, start_neurons)
])
n = np.argmax([
neuron.get_fill_opacity()
for neuron in self.network_mob.layers[-1].neurons
])
rect = SurroundingRectangle(VGroup(
self.network_mob.layers[-1].neurons[n],
self.network_mob.output_labels[n],
))
self.play(ShowCreation(rect))
self.reset_display(rect, image, image_rect)

def reset_display(self, answer_rect, image, image_rect):

self.play(FadeOut(answer_rect))
self.play(
FadeOut(image),
FadeOut(image_rect),
self.network_mob.deactivate_layers,
)

def get_image_to_layer_one_animation(self, image, start_neurons):

image_mover = VGroup(*[
pixel.copy()
for pixel in image
if pixel.fill_rgb[0] > 0.1
])
return Transform(
image_mover, start_neurons,
remover = True,
run_time = 1,
)

###

def add_network(self):
self.network_mob = MNistNetworkMobject(**self.network_mob_config)
self.network = self.network_mob.neural_network
self.add(self.network_mob)

class AlternateNeuralNetworks(PiCreatureScene):

def construct(self):
morty = self.pi_creature
examples = VGroup(
VGroup(
TexText("Convolutional neural network"),
TexText("Good for image recognition"),
),
VGroup(
TexText("Long short-term memory network"),
TexText("Good for speech recognition"),
)
)
for ex in examples:
arrow = Arrow(LEFT, RIGHT, color = BLUE)
ex[0].next_to(arrow, LEFT)
ex[1].next_to(arrow, RIGHT)
ex.submobjects.insert(1, arrow)
examples.set_width(FRAME_WIDTH - 1)
examples.next_to(morty, UP).to_edge(RIGHT)

maybe_words = TexText("Maybe future videos?")

maybe_words.scale(0.8)
maybe_words.next_to(morty, UP)
maybe_words.to_edge(RIGHT)
maybe_words.set_color(YELLOW)

self.play(

Write(examples[0], run_time = 2),
morty.change, "raise_right_hand"
)
self.wait()
self.play(
examples[0].shift, MED_LARGE_BUFF*UP,
FadeIn(examples[1], lag_ratio = 0.5),
)
self.wait()
self.play(
examples.shift, UP,
FadeIn(maybe_words),
morty.change, "maybe"
)
self.wait(2)

class PlainVanillaWrapper(Scene):

def construct(self):
title = TexText("Plain vanilla")
subtitle = TexText("(aka ``multilayer perceptron'')")
title.scale(1.5)
title.to_edge(UP)
subtitle.next_to(title, DOWN)

self.add(title)

self.wait(2)
self.play(Write(subtitle, run_time = 2))
self.wait(2)

class NotPerfectAddOn(Scene):

def construct(self):
words = TexText("Not perfect!")
words.scale(1.5)
arrow = Arrow(UP+RIGHT, DOWN+LEFT, color = RED)
words.set_color(RED)
arrow.to_corner(DOWN+LEFT)
words.next_to(arrow, UP+RIGHT)

self.play(

Write(words),
ShowCreation(arrow),
run_time = 1
)
self.wait(2)

class MoreAThanI(TeacherStudentsScene):

def construct(self):
self.teacher_says(
"More \\\\ A than I",
target_mode = "hesitant"
)
self.change_student_modes("sad", "erm", "tired")
self.wait(2)

class BreakDownName(Scene):

def construct(self):
self.ask_questions()
self.show_neuron()

def ask_questions(self):

name = TexText("Neural", "network")
name.to_edge(UP)
q1 = TexText(
"What are \\\\ the ", "neuron", "s?",
arg_separator = ""
)
q2 = TexText("How are \\\\ they connected?")
q1.next_to(name[0].get_bottom(), DOWN, buff = LARGE_BUFF)
q2.next_to(name[1].get_bottom(), DOWN+RIGHT, buff = LARGE_BUFF)
a1 = Arrow(q1.get_top(), name[0].get_bottom())
a2 = Arrow(q2.get_top(), name.get_corner(DOWN+RIGHT))
VGroup(q1, a1).set_color(BLUE)
VGroup(q2, a2).set_color(YELLOW)

randy = Randolph().to_corner(DOWN+LEFT)

brain = SVGMobject(file_name = "brain")
brain.set_fill(GREY_B, opacity = 0)
brain.replace(randy.eyes, dim_to_match = 1)

self.add(name)

self.play(randy.change, "pondering")
self.play(
brain.set_height, 2,
brain.shift, 2*UP,
brain.set_fill, None, 1,
randy.look, UP
)
brain_outline = brain.copy()
brain_outline.set_fill(opacity = 0)
brain_outline.set_stroke(BLUE_B, 3)
self.play(
ShowPassingFlash(
brain_outline,
time_width = 0.5,
run_time = 2
)
)
self.play(Blink(randy))
self.wait()
self.play(
Write(q1, run_time = 1),
ShowCreation(a1),
name[0].set_color, q1.get_color(),
)
self.play(
Write(q2, run_time = 1),
ShowCreation(a2),
name[1].set_color, q2.get_color()
)
self.wait(2)

self.play(*list(map(FadeOut, [

name, randy, brain,
q2, a1, a2,
q1[0], q1[2]
])))

self.neuron_word = q1[1]

def show_neuron(self):

neuron_word = TexText("Neuron")
arrow = Tex("\\rightarrow")
arrow.shift(LEFT)
description = TexText("Thing that holds a number")
neuron_word.set_color(BLUE)
neuron_word.next_to(arrow, LEFT)
neuron_word.shift(0.5*SMALL_BUFF*UP)
description.next_to(arrow, RIGHT)

neuron = Circle(radius = 0.35, color = BLUE)

neuron.next_to(neuron_word, UP, MED_LARGE_BUFF)
num = Tex("0.2")
num.set_width(0.7*neuron.get_width())
num.move_to(neuron)
num.save_state()
num.move_to(description.get_right())
num.set_fill(opacity = 1)

self.play(

ReplacementTransform(self.neuron_word, neuron_word),
ShowCreation(neuron)
)
self.play(
ShowCreation(arrow),
Write(description, run_time = 1)
)
self.wait()
self.play(
neuron.set_fill, None, 0.2,
num.restore
)
self.wait()
for value in 0.8, 0.4, 0.1, 0.5:
mob = Tex(str(value))
mob.replace(num)
self.play(
neuron.set_fill, None, value,
Transform(num, mob)
)
self.wait()

class IntroduceEachLayer(PreviewMNistNetwork):

CONFIG = {
"network_mob_config" : {
"neuron_stroke_color" : WHITE,
"neuron_stroke_width" : 2,
"neuron_fill_color" : WHITE,
"average_shown_activation_of_large_layer" : False,
"edge_propogation_color" : YELLOW,
"edge_propogation_time" : 2,
}
}
def construct(self):
self.setup_network_mob()
self.break_up_image_as_neurons()
self.show_activation_of_one_neuron()
self.transform_into_full_network()
self.show_output_layer()
self.show_hidden_layers()
self.show_propogation()

def setup_network_mob(self):

self.remove(self.network_mob)

def break_up_image_as_neurons(self):

self.image_map = get_organized_images()
image = self.image_map[9][0]
image_mob = PixelsFromVect(image)
image_mob.set_height(4)
image_mob.next_to(ORIGIN, LEFT)
rect = SurroundingRectangle(image_mob, color = BLUE)
neurons = VGroup()
for pixel in image_mob:
pixel.set_fill(WHITE, opacity = pixel.fill_rgb[0])
neuron = Circle(
color = WHITE,
stroke_width = 1,
radius = pixel.get_width()/2
)
neuron.move_to(pixel)
neuron.set_fill(WHITE, pixel.get_fill_opacity())
neurons.add(neuron)
neurons.scale(1.2)
neurons.space_out_submobjects(1.3)
neurons.to_edge(DOWN)

braces = VGroup(*[Brace(neurons, vect) for vect in (LEFT, UP)])

labels = VGroup(*[
brace.get_tex("28", buff = SMALL_BUFF)
for brace in braces
])

equation = Tex("28", "\\times", "28", "=", "784")

equation.next_to(neurons, RIGHT, LARGE_BUFF, UP)

self.corner_image = MNistMobject(image)

self.corner_image.to_corner(UP+LEFT)

self.add(image_mob, rect)

self.wait()
self.play(
ReplacementTransform(image_mob, neurons),
FadeOut(rect),
FadeIn(braces),
FadeIn(labels),
)
self.wait()
self.play(
ReplacementTransform(labels[0].copy(), equation[0]),
Write(equation[1]),
ReplacementTransform(labels[1].copy(), equation[2]),
Write(equation[3]),
Write(equation[4]),
)
self.wait()

self.neurons = neurons

self.braces = braces
self.brace_labels = labels
self.num_pixels_equation = equation
self.image_vect = image

def show_activation_of_one_neuron(self):

neurons = self.neurons
numbers = VGroup()
example_neuron = None
example_num = None
for neuron in neurons:
o = neuron.get_fill_opacity()
num = DecimalNumber(o, num_decimal_places = 1)
num.set_width(0.7*neuron.get_width())
num.move_to(neuron)
if o > 0.8:
num.set_fill(BLACK)
numbers.add(num)
if o > 0.25 and o < 0.75 and example_neuron is None:
example_neuron = neuron
example_num = num
example_neuron.save_state()
example_num.save_state()
example_neuron.generate_target()
example_neuron.target.set_height(1.5)
example_neuron.target.next_to(neurons, RIGHT)
example_num.target = DecimalNumber(
example_neuron.get_fill_opacity()
)
example_num.target.move_to(example_neuron.target)

def change_activation(num):

self.play(
example_neuron.set_fill, None, num,
ChangingDecimal(
example_num,
lambda a : example_neuron.get_fill_opacity(),
),
UpdateFromFunc(
example_num,
lambda m : m.set_fill(
BLACK if example_neuron.get_fill_opacity() > 0.8 else WHITE
)
)
)

self.play(LaggedStartMap(FadeIn, numbers))

self.play(
MoveToTarget(example_neuron),
MoveToTarget(example_num)
)
self.wait()
curr_opacity = example_neuron.get_fill_opacity()
for num in 0.3, 0.01, 1.0, curr_opacity:
change_activation(num)
self.wait()

rect = SurroundingRectangle(example_num, color = YELLOW)

activation = TexText("``Activation''")
activation.next_to(example_neuron, RIGHT)
activation.set_color(rect.get_color())
self.play(ShowCreation(rect))
self.play(Write(activation, run_time = 1))
self.wait()
change_activation(1.0)
self.wait()
change_activation(0.2)
self.wait()

self.play(

example_neuron.restore,
example_num.restore,
FadeOut(activation),
FadeOut(rect),
)
self.play(FadeOut(numbers))

def transform_into_full_network(self):

network_mob = self.network_mob
neurons = self.neurons
layer = network_mob.layers[0]
layer.save_state()
layer.rotate(np.pi/2)
layer.center()
layer.brace_label.rotate(-np.pi/2)
n = network_mob.max_shown_neurons/2

rows = VGroup(*[

VGroup(*neurons[28*i:28*(i+1)])
for i in range(28)
])

self.play(

FadeOut(self.braces),
FadeOut(self.brace_labels),
FadeOut(VGroup(*self.num_pixels_equation[:-1]))
)

self.play(rows.space_out_submobjects, 1.2)

self.play(
rows.arrange, RIGHT, buff = SMALL_BUFF,
path_arc = np.pi/2,
run_time = 2
)
self.play(
ReplacementTransform(
VGroup(*neurons[:n]),
VGroup(*layer.neurons[:n]),
),
ReplacementTransform(
VGroup(*neurons[n:-n]),
layer.dots,
),
ReplacementTransform(
VGroup(*neurons[-n:]),
VGroup(*layer.neurons[-n:]),
),
)
self.play(
ReplacementTransform(
self.num_pixels_equation[-1],
layer.brace_label
),
FadeIn(layer.brace),
)
self.play(layer.restore, FadeIn(self.corner_image))
self.wait()
for edge_group, layer in zip(network_mob.edge_groups, network_mob.layers[1:]):
self.play(
LaggedStartMap(FadeIn, layer, run_time = 1),
ShowCreation(edge_group),
)
self.wait()

def show_output_layer(self):

layer = self.network_mob.layers[-1]
labels = self.network_mob.output_labels
rect = SurroundingRectangle(
VGroup(layer, labels)
)
neuron = layer.neurons[-1]
neuron.set_fill(WHITE, 0)
label = labels[-1]
for mob in neuron, label:
mob.save_state()
mob.generate_target()
neuron.target.scale(4)
neuron.target.shift(1.5*RIGHT)
label.target.scale(1.5)
label.target.next_to(neuron.target, RIGHT)

activation = DecimalNumber(0)

activation.move_to(neuron.target)

def change_activation(num):

self.play(
neuron.set_fill, None, num,
ChangingDecimal(
activation,
lambda a : neuron.get_fill_opacity(),
),
UpdateFromFunc(
activation,
lambda m : m.set_fill(
BLACK if neuron.get_fill_opacity() > 0.8 else WHITE
)
)
)

self.play(ShowCreation(rect))

self.play(LaggedStartMap(FadeIn, labels))
self.wait()
self.play(
MoveToTarget(neuron),
MoveToTarget(label),
)
self.play(FadeIn(activation))
for num in 0.5, 0.38, 0.97:
change_activation(num)
self.wait()
self.play(
neuron.restore,
neuron.set_fill, None, 1,
label.restore,
FadeOut(activation),
FadeOut(rect),
)
self.wait()

def show_hidden_layers(self):

hidden_layers = VGroup(*self.network_mob.layers[1:3])
rect = SurroundingRectangle(hidden_layers, color = YELLOW)
name = TexText("``Hidden layers''")
name.next_to(rect, UP, SMALL_BUFF)
name.set_color(YELLOW)
q_marks = VGroup()
for layer in hidden_layers:
for neuron in layer.neurons:
q_mark = TexText("?")
q_mark.set_height(0.8*neuron.get_height())
q_mark.move_to(neuron)
q_marks.add(q_mark)
q_marks.set_color_by_gradient(BLUE, YELLOW)
q_mark = TexText("?").scale(4)
q_mark.move_to(hidden_layers)
q_mark.set_color(YELLOW)
q_marks.add(q_mark)

self.play(

ShowCreation(rect),
Write(name)
)
self.wait()
self.play(Write(q_marks))
self.wait()
self.play(
FadeOut(q_marks),
Animation(q_marks[-1].copy())
)

def show_propogation(self):

self.revert_to_original_skipping_status()
self.remove_random_edges(0.7)
self.feed_forward(self.image_vect)

class DiscussChoiceForHiddenLayers(TeacherStudentsScene):

def construct(self):
network_mob = MNistNetworkMobject(
layer_to_layer_buff = 2.5,
neuron_stroke_color = WHITE,
)
network_mob.set_height(4)
network_mob.to_edge(UP, buff = LARGE_BUFF)
layers = VGroup(*network_mob.layers[1:3])
rects = VGroup(*list(map(SurroundingRectangle, layers)))
self.add(network_mob)

two_words = TexText("2 hidden layers")

two_words.set_color(YELLOW)
sixteen_words = TexText("16 neurons each")
sixteen_words.set_color(MAROON_B)
for words in two_words, sixteen_words:
words.next_to(rects, UP)

neurons_anim = LaggedStartMap(

Indicate,
VGroup(*it.chain(*[layer.neurons for layer in layers])),
rate_func = there_and_back,
scale_factor = 2,
color = MAROON_B,
)

self.play(

ShowCreation(rects),
Write(two_words, run_time = 1),
self.teacher.change, "raise_right_hand",
)
self.wait()
self.play(
FadeOut(rects),
ReplacementTransform(two_words, sixteen_words),
neurons_anim
)
self.wait()
self.play(self.teacher.change, "shruggie")
self.change_student_modes("erm", "confused", "sassy")
self.wait()
self.student_says(
"Why 2 \\\\ layers?",
student_index = 1,
bubble_kwargs = {"direction" : RIGHT},
run_time = 1,
target_mode = "raise_left_hand",
)
self.play(self.teacher.change, "happy")
self.wait()
self.student_says(
"Why 16?",
student_index = 0,
run_time = 1,
)
self.play(neurons_anim, run_time = 3)
self.play(
self.teacher.change, "shruggie",
RemovePiCreatureBubble(self.students[0]),
)
self.wait()

class MoreHonestMNistNetworkPreview(IntroduceEachLayer):

CONFIG = {
"network_mob_config" : {
"edge_propogation_time" : 1.5,
}
}
def construct(self):
PreviewMNistNetwork.construct(self)

def get_image_to_layer_one_animation(self, image, start_neurons):

neurons = VGroup()
for pixel in image:
neuron = Circle(
radius = pixel.get_width()/2,
stroke_width = 1,
stroke_color = WHITE,
fill_color = WHITE,
fill_opacity = pixel.fill_rgb[0]
)
neuron.move_to(pixel)
neurons.add(neuron)
neurons.scale(1.2)
neurons.next_to(image, DOWN)
n = len(start_neurons)
point = VectorizedPoint(start_neurons.get_center())
target = VGroup(*it.chain(
start_neurons[:n/2],
[point.copy() for x in range(len(neurons)-n)],
start_neurons[n/2:],
))
mover = image.copy()
self.play(Transform(mover, neurons))
return Transform(
mover, target,
run_time = 2,
lag_ratio = 0.5,
remover = True
)

class AskAboutPropogationAndTraining(TeacherStudentsScene):

def construct(self):
self.student_says(
"How does one layer \\\\ influence the next?",
student_index = 0,
run_time = 1
)
self.wait()
self.student_says(
"How does \\\\ training work?",
student_index = 2,
run_time = 1
)
self.wait(3)

class AskAboutLayers(PreviewMNistNetwork):

def construct(self):
self.play(
self.network_mob.scale, 0.8,
self.network_mob.to_edge, DOWN,
)

question = TexText("Why the", "layers?")

question.to_edge(UP)
neuron_groups = [
layer.neurons
for layer in self.network_mob.layers
]
arrows = VGroup(*[
Arrow(
question[1].get_bottom(),
group.get_top()
)
for group in neuron_groups
])
rects = list(map(SurroundingRectangle, neuron_groups[1:3]))

self.play(

Write(question, run_time = 1),
LaggedStartMap(
GrowFromPoint, arrows,
lambda a : (a, a.get_start()),
run_time = 2
)
)
self.wait()
self.play(*list(map(ShowCreation, rects)))
self.wait()

class BreakUpMacroPatterns(IntroduceEachLayer):

CONFIG = {
"camera_config" : {"background_opacity" : 1},
"prefixes" : [
"nine", "eight", "four",
"upper_loop", "right_line",
"lower_loop", "horizontal_line",
"upper_left_line"
]
}
def construct(self):
self.setup_network_mob()
self.setup_needed_patterns()
self.setup_added_patterns()
self.show_nine()
self.show_eight()
self.show_four()
self.show_second_to_last_layer()
self.show_upper_loop_activation()
self.show_what_learning_is_required()

def setup_needed_patterns(self):

prefixes = self.prefixes
vects = [
np.array(Image.open(
get_full_raster_image_path("handwritten_" + p),
))[:,:,0].flatten()/255.0
for p in prefixes
]
mobjects = list(map(MNistMobject, vects))
for mob in mobjects:
image = mob[1]
self.make_transparent(image)
for prefix, mob in zip(prefixes, mobjects):
setattr(self, prefix, mob)

def setup_added_patterns(self):

image_map = get_organized_images()
two, three, five = mobs = [
MNistMobject(image_map[n][0])
for n in (2, 3, 5)
]
self.added_patterns = VGroup()
for mob in mobs:
for i, j in it.product([0, 14], [0, 14]):
pattern = mob.deepcopy()
pa = pattern[1].pixel_array
temp = np.array(pa[i:i+14,j:j+14,:], dtype = 'uint8')
pa[:,:] = 0
pa[i:i+14,j:j+14,:] = temp
self.make_transparent(pattern[1])
pattern[1].set_color(random_bright_color())
self.added_patterns.add(pattern)
self.image_map = image_map

def show_nine(self):

nine = self.nine
upper_loop = self.upper_loop
right_line = self.right_line
equation = self.get_equation(nine, upper_loop, right_line)
equation.to_edge(UP)
equation.shift(LEFT)

parts = [upper_loop[1], right_line[1]]

for mob, color in zip(parts, [YELLOW, RED]):
mob.set_color(color)
mob.save_state()
mob.move_to(nine)
right_line[1].pixel_array[:14,:,3] = 0

self.play(FadeIn(nine))

self.wait()
self.play(*list(map(FadeIn, parts)))
self.wait()
self.play(
Write(equation[1]),
upper_loop[1].restore,
FadeIn(upper_loop[0])
)
self.wait()
self.play(
Write(equation[3]),
right_line[1].restore,
FadeIn(right_line[0]),
)
self.wait()

self.nine_equation = equation

def show_eight(self):

eight = self.eight
upper_loop = self.upper_loop.deepcopy()
lower_loop = self.lower_loop
lower_loop[1].set_color(GREEN)

equation = self.get_equation(eight, upper_loop, lower_loop)

equation.next_to(self.nine_equation, DOWN)

lower_loop[1].save_state()

lower_loop[1].move_to(eight[1])

self.play(

FadeIn(eight),
Write(equation[1]),
)
self.play(ReplacementTransform(
self.upper_loop.copy(),
upper_loop
))
self.wait()
self.play(FadeIn(lower_loop[1]))
self.play(
Write(equation[3]),
lower_loop[1].restore,
FadeIn(lower_loop[0]),
)
self.wait()

self.eight_equation = equation

def show_four(self):

four = self.four
upper_left_line = self.upper_left_line
upper_left_line[1].set_color(BLUE)
horizontal_line = self.horizontal_line
horizontal_line[1].set_color(MAROON_B)
right_line = self.right_line.deepcopy()
equation = self.get_equation(four, right_line, upper_left_line, horizontal_line)
equation.next_to(
self.eight_equation, DOWN, aligned_edge = LEFT
)

self.play(

FadeIn(four),
Write(equation[1])
)
self.play(ReplacementTransform(
self.right_line.copy(), right_line
))
self.play(LaggedStartMap(
FadeIn, VGroup(*equation[3:])
))
self.wait(2)

self.four_equation = equation

def show_second_to_last_layer(self):

everything = VGroup(*it.chain(
self.nine_equation,
self.eight_equation,
self.four_equation,
))
patterns = VGroup(
self.upper_loop,
self.lower_loop,
self.right_line,
self.upper_left_line,
self.horizontal_line,
*self.added_patterns[:11]
)
for pattern in patterns:
pattern.add_to_back(
pattern[1].copy().set_color(BLACK, alpha = 1)
)
everything.remove(*patterns)
network_mob = self.network_mob
layer = network_mob.layers[-2]
patterns.generate_target()
for pattern, neuron in zip(patterns.target, layer.neurons):
pattern.set_height(neuron.get_height())
pattern.next_to(neuron, RIGHT, SMALL_BUFF)
for pattern in patterns[5:]:
pattern.fade(1)

self.play(*list(map(FadeOut, everything)))

self.play(
FadeIn(
network_mob,
lag_ratio = 0.5,
run_time = 3,
),
MoveToTarget(patterns)
)
self.wait(2)

self.patterns = patterns

def show_upper_loop_activation(self):

neuron = self.network_mob.layers[-2].neurons[0]
words = TexText("Upper loop neuron...maybe...")
words.scale(0.8)
words.next_to(neuron, UP)
words.shift(RIGHT)
rect = SurroundingRectangle(VGroup(
neuron, self.patterns[0]
))
nine = self.nine
upper_loop = self.upper_loop.copy()
upper_loop.remove(upper_loop[0])
upper_loop.replace(nine)
nine.add(upper_loop)
nine.to_corner(UP+LEFT)
self.remove_random_edges(0.7)
self.network.get_activation_of_all_layers = lambda v : [
np.zeros(784),
sigmoid(6*(np.random.random(16)-0.5)),
np.array([1, 0, 1] + 13*[0]),
np.array(9*[0] + [1])
]

self.play(FadeIn(nine))

self.play(
ShowCreation(rect),
Write(words)
)
self.add_foreground_mobject(self.patterns)
self.feed_forward(np.random.random(784))
self.wait(2)

def show_what_learning_is_required(self):

edge_group = self.network_mob.edge_groups[-1].copy()
edge_group.set_stroke(YELLOW, 4)
for x in range(3):
self.play(LaggedStartMap(
ShowCreationThenDestruction, edge_group,
run_time = 3
))
self.wait()

######

def get_equation(self, *mobs):
equation = VGroup(
mobs[0], Tex("=").scale(2),
*list(it.chain(*[
[m, Tex("+").scale(2)]
for m in mobs[1:-1]
])) + [mobs[-1]]
)
equation.arrange(RIGHT)
return equation

def make_transparent(self, image_mob):

return make_transparent(image_mob)
alpha_vect = np.array(
image_mob.pixel_array[:,:,0],
dtype = 'uint8'
)
image_mob.set_color(WHITE)
image_mob.pixel_array[:,:,3] = alpha_vect
return image_mob

class GenerallyLoopyPattern(Scene):

def construct(self):
image_map = get_organized_images()
images = list(map(MNistMobject, it.chain(
image_map[8], image_map[9],
)))
random.shuffle(images)

for image in images:

image.to_corner(DOWN+RIGHT)
self.add(image)
self.wait(0.2)
self.remove(image)

class HowWouldYouRecognizeSubcomponent(TeacherStudentsScene):

def construct(self):
self.student_says(
"Okay, but recognizing loops \\\\",
"is just as hard!",
target_mode = "sassy"
)
self.play(
self.teacher.change, "guilty"
)
self.wait()

class BreakUpMicroPatterns(BreakUpMacroPatterns):

CONFIG = {
"prefixes" : [
"loop",
"loop_edge1",
"loop_edge2",
"loop_edge3",
"loop_edge4",
"loop_edge5",
"right_line",
"right_line_edge1",
"right_line_edge2",
"right_line_edge3",
]
}
def construct(self):
self.setup_network_mob()
self.setup_needed_patterns()

self.break_down_loop()

self.break_down_long_line()

def break_down_loop(self):

loop = self.loop
loop[0].set_color(WHITE)
edges = Group(*[
getattr(self, "loop_edge%d"%d)
for d in range(1, 6)
])
colors = color_gradient([BLUE, YELLOW, RED], 5)
for edge, color in zip(edges, colors):
for mob in edge:
mob.set_color(color)
loop.generate_target()
edges.generate_target()
for edge in edges:
edge[0].set_stroke(width = 0)
edge.save_state()
edge[1].set_opacity(0)
equation = self.get_equation(loop.target, *edges.target)
equation.set_width(FRAME_WIDTH - 1)
equation.to_edge(UP)
symbols = VGroup(*equation[1::2])

randy = Randolph()

randy.to_corner(DOWN+LEFT)

self.add(randy)

self.play(
FadeIn(loop),
randy.change, "pondering", loop
)
self.play(Blink(randy))
self.wait()
self.play(LaggedStartMap(
ApplyMethod, edges,
lambda e : (e.restore,),
run_time = 4
))
self.wait()
self.play(
MoveToTarget(loop, run_time = 2),
MoveToTarget(edges, run_time = 2),
Write(symbols),
randy.change, "happy", equation,
)
self.wait()

self.loop_equation = equation

self.randy = randy

def break_down_long_line(self):

randy = self.randy
line = self.right_line
line[0].set_color(WHITE)
edges = Group(*[
getattr(self, "right_line_edge%d"%d)
for d in range(1, 4)
])
colors = Color(MAROON_B).range_to(PURPLE, 3)
for edge, color in zip(edges, colors):
for mob in edge:
mob.set_color(color)
equation = self.get_equation(line, *edges)
equation.set_height(self.loop_equation.get_height())
equation.next_to(
self.loop_equation, DOWN, MED_LARGE_BUFF, LEFT
)
image_map = get_organized_images()
digits = VGroup(*[
MNistMobject(image_map[n][1])
for n in (1, 4, 7)
])
digits.arrange(RIGHT)
digits.next_to(randy, RIGHT)

self.revert_to_original_skipping_status()

self.play(
FadeIn(line),
randy.change, "hesitant", line
)
self.play(Blink(randy))
self.play(LaggedStartMap(FadeIn, digits))
self.wait()
self.play(
LaggedStartMap(FadeIn, Group(*equation[1:])),
randy.change, "pondering", equation
)
self.wait(3)

class SecondLayerIsLittleEdgeLayer(IntroduceEachLayer):

CONFIG = {
"camera_config" : {
"background_opacity" : 1,
},
"network_mob_config" : {
"layer_to_layer_buff" : 2,
"edge_propogation_color" : YELLOW,
}
}
def construct(self):
self.setup_network_mob()
self.setup_activations_and_nines()

self.describe_second_layer()

self.show_propogation()
self.ask_question()

def setup_network_mob(self):

self.network_mob.scale(0.7)
self.network_mob.to_edge(DOWN)
self.remove_random_edges(0.7)

def setup_activations_and_nines(self):

layers = self.network_mob.layers
nine_im, loop_im, line_im = images = [
Image.open(get_full_raster_image_path("handwritten_%s"%s))
for s in ("nine", "upper_loop", "right_line")
]
nine_array, loop_array, line_array = [
np.array(im)[:,:,0]/255.0
for im in images
]
self.nine = MNistMobject(nine_array.flatten())
self.nine.set_height(1.5)
self.nine[0].set_color(WHITE)
make_transparent(self.nine[1])
self.nine.next_to(layers[0].neurons, UP)

self.activations = self.network.get_activation_of_all_layers(

nine_array.flatten()
)
self.activations[-2] = np.array([1, 0, 1] + 13*[0])
self.edge_colored_nine = Group()
nine_pa = self.nine[1].pixel_array
n, k = 6, 4
colors = color_gradient([BLUE, YELLOW, RED, MAROON_B, GREEN], 10)
for i, j in it.product(list(range(n)), list(range(k))):
mob = ImageMobject(np.zeros((28, 28, 4), dtype = 'uint8'))
mob.replace(self.nine[1])
pa = mob.pixel_array
color = colors[(k*i + j)%(len(colors))]
rgb = (255*color_to_rgb(color)).astype('uint8')
pa[:,:,:3] = rgb
i0, i1 = 1+(28/n)*i, 1+(28/n)*(i+1)
j0, j1 = (28/k)*j, (28/k)*(j+1)
pa[i0:i1,j0:j1,3] = nine_pa[i0:i1,j0:j1,3]
self.edge_colored_nine.add(mob)
self.edge_colored_nine.next_to(layers[1], UP)

loop, line = [

ImageMobject(layer_to_image_array(array.flatten()))
for array in (loop_array, line_array)
]
for mob, color in (loop, YELLOW), (line, RED):
make_transparent(mob)
mob.set_color(color)
mob.replace(self.nine[1])
line.pixel_array[:14,:,:] = 0

self.pattern_colored_nine = Group(loop, line)

self.pattern_colored_nine.next_to(layers[2], UP)

for mob in self.edge_colored_nine, self.pattern_colored_nine:

mob.align_to(self.nine[1], UP)

def describe_second_layer(self):

layer = self.network_mob.layers[1]
rect = SurroundingRectangle(layer)
words = TexText("``Little edge'' layer?")
words.next_to(rect, UP, MED_LARGE_BUFF)
words.set_color(YELLOW)

self.play(

ShowCreation(rect),
Write(words, run_time = 2)
)
self.wait()
self.play(*list(map(FadeOut, [rect, words])))

def show_propogation(self):

nine = self.nine
edge_colored_nine = self.edge_colored_nine
pattern_colored_nine = self.pattern_colored_nine
activations = self.activations
network_mob = self.network_mob
layers = network_mob.layers
edge_groups = network_mob.edge_groups.copy()
edge_groups.set_stroke(YELLOW, 4)

v_nine = PixelsAsSquares(nine[1])

neurons = VGroup()
for pixel in v_nine:
neuron = Circle(
radius = pixel.get_width()/2,
stroke_color = WHITE,
stroke_width = 1,
fill_color = WHITE,
fill_opacity = pixel.get_fill_opacity(),
)
neuron.rotate(3*np.pi/4)
neuron.move_to(pixel)
neurons.add(neuron)
neurons.set_height(2)
neurons.space_out_submobjects(1.2)
neurons.next_to(network_mob, LEFT)
self.set_neurons_target(neurons, layers[0])

pattern_colored_nine.save_state()

pattern_colored_nine.move_to(edge_colored_nine)
edge_colored_nine.save_state()
edge_colored_nine.move_to(nine[1])
for mob in edge_colored_nine, pattern_colored_nine:
for submob in mob:
submob.set_opacity(0)

active_layers = [

network_mob.get_active_layer(i, a)
for i, a in enumerate(activations)
]

def activate_layer(i):

self.play(
ShowCreationThenDestruction(
edge_groups[i-1],
run_time = 2,
lag_ratio = 0.5
),
FadeIn(active_layers[i])
)
self.play(FadeIn(nine))
self.play(ReplacementTransform(v_nine, neurons))
self.play(MoveToTarget(
neurons,
remover = True,
lag_ratio = 0.5,
run_time = 2
))

activate_layer(1)

self.play(edge_colored_nine.restore)
self.separate_parts(edge_colored_nine)
self.wait()

activate_layer(2)

self.play(pattern_colored_nine.restore)
self.separate_parts(pattern_colored_nine)

activate_layer(3)

self.wait(2)

def ask_question(self):

question = TexText(
"Does the network \\\\ actually do this?"
)
question.to_edge(LEFT)
later = TexText("We'll get back \\\\ to this")
later.to_corner(UP+LEFT)
later.set_color(BLUE)
arrow = Arrow(later.get_bottom(), question.get_top())
arrow.set_color(BLUE)

self.play(Write(question, run_time = 2))

self.wait()
self.play(
FadeIn(later),
GrowFromPoint(arrow, arrow.get_start())
)
self.wait()

###

def set_neurons_target(self, neurons, layer):
neurons.generate_target()
n = len(layer.neurons)/2
Transform(
VGroup(*neurons.target[:n]),
VGroup(*layer.neurons[:n]),
).update(1)
Transform(
VGroup(*neurons.target[-n:]),
VGroup(*layer.neurons[-n:]),
).update(1)
Transform(
VGroup(*neurons.target[n:-n]),
VectorizedPoint(layer.get_center())
).update(1)

def separate_parts(self, image_group):

vects = compass_directions(len(image_group), UP)
image_group.generate_target()
for im, vect in zip(image_group.target, vects):
im.shift(MED_SMALL_BUFF*vect)
self.play(MoveToTarget(
image_group,
rate_func = there_and_back,
lag_ratio = 0.5,
run_time = 2,
))

class EdgeDetection(Scene):

CONFIG = {
"camera_config" : {"background_opacity" : 1}
}
def construct(self):
lion = ImageMobject("Lion")
edges_array = get_edges(lion.pixel_array)
edges = ImageMobject(edges_array)
group = Group(lion, edges)
group.set_height(4)
group.arrange(RIGHT)
lion_copy = lion.copy()

self.play(FadeIn(lion))

self.play(lion_copy.move_to, edges)
self.play(Transform(lion_copy, edges, run_time = 3))
self.wait(2)

class ManyTasksBreakDownLikeThis(TeacherStudentsScene):

def construct(self):
audio = self.get_wave_form()
audio_label = TexText("Raw audio")
letters = TexText(" ".join("recognition"))
syllables = TexText("$\\cdot$".join([
"re", "cog", "ni", "tion"
]))
word = TexText(
"re", "cognition",
arg_separator = ""
)
word[1].set_color(BLUE)
arrows = VGroup()
def get_arrow():
arrow = Arrow(ORIGIN, RIGHT, color = BLUE)
arrows.add(arrow)
return arrow
sequence = VGroup(
audio, get_arrow(),
letters, get_arrow(),
syllables, get_arrow(),
word
)
sequence.arrange(RIGHT)
sequence.set_width(FRAME_WIDTH - 1)
sequence.to_edge(UP)

audio_label.next_to(audio, DOWN)

VGroup(audio, audio_label).set_color(YELLOW)
audio.save_state()

self.teacher_says(

"Many", "recognition", "tasks\\\\",
"break down like this"
)
self.change_student_modes(*["pondering"]*3)
self.wait()
content = self.teacher.bubble.content
pre_word = content[1]
content.remove(pre_word)
audio.move_to(pre_word)
self.play(
self.teacher.bubble.content.fade, 1,
ShowCreation(audio),
pre_word.shift, MED_SMALL_BUFF, DOWN
)
self.wait(2)
self.play(
RemovePiCreatureBubble(self.teacher),
audio.restore,
FadeIn(audio_label),
*[
ReplacementTransform(
m1, m2
)
for m1, m2 in zip(pre_word, letters)
]
)
self.play(
GrowFromPoint(arrows[0], arrows[0].get_start()),
)
self.wait()
self.play(
GrowFromPoint(arrows[1], arrows[1].get_start()),
LaggedStartMap(FadeIn, syllables, run_time = 1)
)
self.wait()
self.play(
GrowFromPoint(arrows[2], arrows[2].get_start()),
LaggedStartMap(FadeIn, word, run_time = 1)
)
self.wait()

def get_wave_form(self):

func = lambda x : abs(sum([
(1./n)*np.sin((n+3)*x)
for n in range(1, 5)
]))
result = VGroup(*[
Line(func(x)*DOWN, func(x)*UP)
for x in np.arange(0, 4, 0.1)
])
result.set_stroke(width = 2)
result.arrange(RIGHT, buff = MED_SMALL_BUFF)
result.set_height(1)

return result

class AskAboutWhatEdgesAreDoing(IntroduceEachLayer):

CONFIG = {
"network_mob_config" : {
"layer_to_layer_buff" : 2,
}
}
def construct(self):
self.add_question()
self.show_propogation()

def add_question(self):

self.network_mob.scale(0.8)
self.network_mob.to_edge(DOWN)
edge_groups = self.network_mob.edge_groups
self.remove_random_edges(0.7)

question = TexText(

"What are these connections actually doing?"
)
question.to_edge(UP)
question.shift(RIGHT)
arrows = VGroup(*[
Arrow(
question.get_bottom(),
edge_group.get_top()
)
for edge_group in edge_groups
])

self.add(question, arrows)

def show_propogation(self):

in_vect = get_organized_images()[6][3]
image = MNistMobject(in_vect)
image.next_to(self.network_mob, LEFT, MED_SMALL_BUFF, UP)

self.add(image)

self.feed_forward(in_vect)
self.wait()

class IntroduceWeights(IntroduceEachLayer):

CONFIG = {
"weights_color" : GREEN,
"negative_weights_color" : RED,
}
def construct(self):
self.zoom_in_on_one_neuron()
self.show_desired_pixel_region()
self.ask_about_parameters()
self.show_weights()
self.show_weighted_sum()
self.organize_weights_as_grid()
self.make_most_weights_0()
self.add_negative_weights_around_the_edge()

def zoom_in_on_one_neuron(self):

self.network_mob.to_edge(LEFT)
layers = self.network_mob.layers
edge_groups = self.network_mob.edge_groups

neuron = layers[1].neurons[7].deepcopy()

self.play(

FadeOut(edge_groups),
FadeOut(VGroup(*layers[1:])),
FadeOut(self.network_mob.output_labels),
Animation(neuron),
neuron.edges_in.set_stroke, None, 2,
lag_ratio = 0.5,
run_time = 2
)

self.neuron = neuron

def show_desired_pixel_region(self):

neuron = self.neuron
d = 28

pixels = PixelsAsSquares(ImageMobject(

np.zeros((d, d, 4))
))
pixels.set_stroke(width = 0.5)
pixels.set_fill(WHITE, 0)
pixels.set_height(4)
pixels.next_to(neuron, RIGHT, LARGE_BUFF)
rect = SurroundingRectangle(pixels, color = BLUE)

pixels_to_detect = self.get_pixels_to_detect(pixels)

self.play(

FadeIn(rect),
ShowCreation(
pixels,
lag_ratio = 0.5,
run_time = 2,
)
)
self.play(
pixels_to_detect.set_fill, WHITE, 1,
lag_ratio = 0.5,
run_time = 2
)
self.wait(2)

self.pixels = pixels

self.pixels_to_detect = pixels_to_detect
self.pixels_group = VGroup(rect, pixels)

def ask_about_parameters(self):

pixels = self.pixels
pixels_group = self.pixels_group
neuron = self.neuron

question = TexText("What", "parameters", "should exist?")

parameter_word = question.get_part_by_tex("parameters")
parameter_word.set_color(self.weights_color)
question.move_to(neuron.edges_in.get_top(), LEFT)
arrow = Arrow(
parameter_word.get_bottom(),
neuron.edges_in[0].get_center(),
color = self.weights_color
)

p_labels = VGroup(*[

Tex("p_%d\\!:"%(i+1)).set_color(self.weights_color)
for i in range(8)
] + [Tex("\\vdots")])
p_labels.arrange(DOWN, aligned_edge = LEFT)
p_labels.next_to(parameter_word, DOWN, LARGE_BUFF)
p_labels[-1].shift(SMALL_BUFF*RIGHT)

def get_alpha_func(i, start = 0):

# m = int(5*np.sin(2*np.pi*i/128.))
m = random.randint(1, 10)
return lambda a : start + (1-2*start)*np.sin(np.pi*a*m)**2

decimals = VGroup()

changing_decimals = []
for i, p_label in enumerate(p_labels[:-1]):
decimal = DecimalNumber(0)
decimal.next_to(p_label, RIGHT, MED_SMALL_BUFF)
decimals.add(decimal)
changing_decimals.append(ChangingDecimal(
decimal, get_alpha_func(i + 5)
))
for i, pixel in enumerate(pixels):
pixel.func = get_alpha_func(i, pixel.get_fill_opacity())
pixel_updates = [
UpdateFromAlphaFunc(
pixel,
lambda p, a : p.set_fill(opacity = p.func(a))
)
for pixel in pixels
]

self.play(

Write(question, run_time = 2),
GrowFromPoint(arrow, arrow.get_start()),
pixels_group.set_height, 3,
pixels_group.to_edge, RIGHT,
LaggedStartMap(FadeIn, p_labels),
LaggedStartMap(FadeIn, decimals),
)
self.wait()
self.play(
*changing_decimals + pixel_updates,
run_time = 5,
rate_func=linear
)

self.question = question

self.weight_arrow = arrow
self.p_labels = p_labels
self.decimals = decimals

def show_weights(self):

p_labels = self.p_labels
decimals = self.decimals
arrow = self.weight_arrow
question = self.question
neuron = self.neuron
edges = neuron.edges_in

parameter_word = question.get_part_by_tex("parameters")

question.remove(parameter_word)
weights_word = TexText("Weights", "")[0]
weights_word.set_color(self.weights_color)
weights_word.move_to(parameter_word)

w_labels = VGroup()

for p_label in p_labels:
w_label = Tex(
p_label.get_tex().replace("p", "w")
)
w_label.set_color(self.weights_color)
w_label.move_to(p_label)
w_labels.add(w_label)

edges.generate_target()

random_numbers = 1.5*np.random.random(len(edges))-0.5
self.make_edges_weighted(edges.target, random_numbers)
def get_alpha_func(r):
return lambda a : (4*r)*a

self.play(

FadeOut(question),
ReplacementTransform(parameter_word, weights_word),
ReplacementTransform(p_labels, w_labels)
)
self.play(
MoveToTarget(edges),
*[
ChangingDecimal(
decimal,
get_alpha_func(r)
)
for decimal, r in zip(decimals, random_numbers)
]
)
self.play(LaggedStartMap(
ApplyMethod, edges,
lambda m : (m.rotate, np.pi/24),
rate_func = wiggle,
run_time = 2
))
self.wait()

self.w_labels = w_labels

self.weights_word = weights_word
self.random_numbers = random_numbers

def show_weighted_sum(self):

weights_word = self.weights_word
weight_arrow = self.weight_arrow
w_labels = VGroup(*[
VGroup(*label[:-1]).copy()
for label in self.w_labels
])
layer = self.network_mob.layers[0]

a_vect = np.random.random(16)

active_layer = self.network_mob.get_active_layer(0, a_vect)

a_labels = VGroup(*[

Tex("a_%d"%d)
for d in range(1, 5)
])

weighted_sum = VGroup(*it.chain(*[

[w, a, Tex("+")]
for w, a in zip(w_labels, a_labels)
]))
weighted_sum.add(
Tex("\\cdots"),
Tex("+"),
Tex("w_n").set_color(self.weights_color),
Tex("a_n")
)
weighted_sum.arrange(RIGHT, buff = SMALL_BUFF)
weighted_sum.to_edge(UP)

self.play(Transform(layer, active_layer))

self.play(
FadeOut(weights_word),
FadeOut(weight_arrow),
*[
ReplacementTransform(n.copy(), a)
for n, a in zip(layer.neurons, a_labels)
] + [
ReplacementTransform(n.copy(), weighted_sum[-4])
for n in layer.neurons[4:-1]
] + [
ReplacementTransform(
layer.neurons[-1].copy(),
weighted_sum[-1]
)
] + [
Write(weighted_sum[i])
for i in list(range(2, 12, 3)) + [-4, -3]
],
run_time = 1.5
)
self.wait()
self.play(*[
ReplacementTransform(w1.copy(), w2)
for w1, w2 in zip(self.w_labels, w_labels)[:4]
]+[
ReplacementTransform(w.copy(), weighted_sum[-4])
for w in self.w_labels[4:-1]
]+[
ReplacementTransform(
self.w_labels[-1].copy(), weighted_sum[-2]
)
], run_time = 2)
self.wait(2)

self.weighted_sum = weighted_sum

def organize_weights_as_grid(self):

pixels = self.pixels
w_labels = self.w_labels
decimals = self.decimals

weights = 2*np.sqrt(np.random.random(784))-1

weights[:8] = self.random_numbers[:8]
weights[-8:] = self.random_numbers[-8:]

weight_grid = PixelsFromVect(np.abs(weights))

weight_grid.replace(pixels)
weight_grid.next_to(pixels, LEFT)
for weight, pixel in zip(weights, weight_grid):
if weight >= 0:
color = self.weights_color
else:
color = self.negative_weights_color
pixel.set_fill(color, opacity = abs(weight))

self.play(FadeOut(w_labels))

self.play(
FadeIn(
VGroup(*weight_grid[len(decimals):]),
lag_ratio = 0.5,
run_time = 3
),
*[
ReplacementTransform(decimal, pixel)
for decimal, pixel in zip(decimals, weight_grid)
]
)
self.wait()

self.weight_grid = weight_grid

def make_most_weights_0(self):

weight_grid = self.weight_grid
pixels = self.pixels
pixels_group = self.pixels_group

weight_grid.generate_target()

for w, p in zip(weight_grid.target, pixels):
if p.get_fill_opacity() > 0.1:
w.set_fill(GREEN, 0.5)
else:
w.set_fill(BLACK, 0.5)
w.set_stroke(WHITE, 0.5)

digit = self.get_digit()

digit.replace(pixels)

self.play(MoveToTarget(

weight_grid,
run_time = 2,
lag_ratio = 0.5
))
self.wait()
self.play(Transform(
pixels, digit,
run_time = 2,
lag_ratio = 0.5
))
self.wait()
self.play(weight_grid.move_to, pixels)
self.wait()
self.play(
ReplacementTransform(
self.pixels_to_detect.copy(),
self.weighted_sum,
run_time = 3,
lag_ratio = 0.5
),
Animation(weight_grid),
)
self.wait()

def add_negative_weights_around_the_edge(self):

weight_grid = self.weight_grid
pixels = self.pixels

self.play(weight_grid.next_to, pixels, LEFT)

self.play(*[
ApplyMethod(
weight_grid[28*y + x].set_fill,
self.negative_weights_color,
0.5
)
for y in (6, 10)
for x in range(14-4, 14+4)
])
self.wait(2)
self.play(weight_grid.move_to, pixels)
self.wait(2)

####

def get_digit(self):
digit_vect = get_organized_images()[7][4]
digit = PixelsFromVect(digit_vect)
digit.set_stroke(width = 0.5)
return digit

def get_pixels_to_detect(self, pixels):

d = int(np.sqrt(len(pixels)))
return VGroup(*it.chain(*[
pixels[d*n + d/2 - 4 : d*n + d/2 + 4]
for n in range(7, 10)
]))

def get_surrounding_pixels_for_edge(self, pixels):

d = int(np.sqrt(len(pixels)))
return VGroup(*it.chain(*[
pixels[d*n + d/2 - 4 : d*n + d/2 + 4]
for n in (6, 10)
]))

def make_edges_weighted(self, edges, weights):

for edge, r in zip(edges, weights):
if r > 0:
color = self.weights_color
else:
color = self.negative_weights_color
edge.set_stroke(color, 6*abs(r))

class MotivateSquishing(Scene):

def construct(self):
self.add_weighted_sum()
self.show_real_number_line()
self.show_interval()
self.squish_into_interval()

def add_weighted_sum(self):

weighted_sum = Tex(*it.chain(*[
["w_%d"%d, "a_%d"%d, "+"]
for d in range(1, 5)
] + [
["\\cdots", "+", "w_n", "a_n"]
]))
weighted_sum.set_color_by_tex("w_", GREEN)
weighted_sum.to_edge(UP)
self.add(weighted_sum)
self.weighted_sum = weighted_sum

def show_real_number_line(self):

weighted_sum = self.weighted_sum
number_line = NumberLine(unit_size = 1.5)
number_line.add_numbers()
number_line.shift(UP)
arrow1, arrow2 = [
Arrow(
weighted_sum.get_bottom(),
number_line.number_to_point(n),
)
for n in (-3, 3)
]

self.play(Write(number_line))

self.play(GrowFromPoint(arrow1, arrow1.get_start()))
self.play(Transform(
arrow1, arrow2,
run_time = 5,
rate_func = there_and_back
))
self.play(FadeOut(arrow1))

self.number_line = number_line

def show_interval(self):

lower_number_line = self.number_line.copy()
lower_number_line.shift(2*DOWN)
lower_number_line.set_color(GREY_B)
lower_number_line.numbers.set_color(WHITE)
interval = Line(
lower_number_line.number_to_point(0),
lower_number_line.number_to_point(1),
color = YELLOW,
stroke_width = 5
)
brace = Brace(interval, DOWN, buff = 0.7)
words = TexText("Activations should be in this range")
words.next_to(brace, DOWN, SMALL_BUFF)

self.play(ReplacementTransform(

self.number_line.copy(), lower_number_line
))
self.play(
GrowFromCenter(brace),
GrowFromCenter(interval),
)
self.play(Write(words, run_time = 2))
self.wait()

self.lower_number_line = lower_number_line

def squish_into_interval(self):

line = self.number_line
line.remove(*line.numbers)
ghost_line = line.copy()
ghost_line.fade(0.5)
ghost_line.set_color(BLUE_E)
self.add(ghost_line, line)
lower_line = self.lower_number_line

line.generate_target()

u = line.unit_size
line.target.apply_function(
lambda p : np.array([u*sigmoid(p[0])]+list(p[1:]))
)
line.target.move_to(lower_line.number_to_point(0.5))

arrow = Arrow(

line.numbers.get_bottom(),
line.target.get_top(),
color = YELLOW
)

self.play(

MoveToTarget(line),
GrowFromPoint(arrow, arrow.get_start())
)
self.wait(2)

class IntroduceSigmoid(GraphScene):

CONFIG = {
"x_min" : -5,
"x_max" : 5,
"x_axis_width" : 12,
"y_min" : -1,
"y_max" : 2,
"y_axis_label" : "",
"graph_origin" : DOWN,
"x_labeled_nums" : list(range(-4, 5)),
"y_labeled_nums" : list(range(-1, 3)),
}
def construct(self):
self.setup_axes()
self.add_title()
self.add_graph()
self.show_part(-5, -2, RED)
self.show_part(2, 5, GREEN)
self.show_part(-2, 2, BLUE)

def add_title(self):

name = TexText("Sigmoid")
name.next_to(ORIGIN, RIGHT, LARGE_BUFF)
name.to_edge(UP)
char = self.x_axis_label.replace("$", "")
equation = Tex(
"\\sigma(%s) = \\frac{1}{1+e^{-%s}}"%(char, char)
)
equation.next_to(name, DOWN)
self.add(equation, name)

self.equation = equation

self.sigmoid_name = name

def add_graph(self):

graph = self.get_graph(
lambda x : 1./(1+np.exp(-x)),
color = YELLOW
)

self.play(ShowCreation(graph))

self.wait()

self.sigmoid_graph = graph

###

def show_part(self, x_min, x_max, color):
line, graph_part = [
self.get_graph(
func,
x_min = x_min,
x_max = x_max,
color = color,
).set_stroke(width = 4)
for func in (lambda x : 0, sigmoid)
]

self.play(ShowCreation(line))

self.wait()
self.play(Transform(line, graph_part))
self.wait()

class IncludeBias(IntroduceWeights):

def construct(self):
self.force_skipping()
self.zoom_in_on_one_neuron()
self.setup_start()
self.revert_to_original_skipping_status()

self.add_sigmoid_label()

self.words_on_activation()
self.comment_on_need_for_bias()
self.add_bias()
self.summarize_weights_and_biases()

def setup_start(self):

self.weighted_sum = self.get_weighted_sum()
digit = self.get_digit()
rect = SurroundingRectangle(digit)
d_group = VGroup(digit, rect)
d_group.set_height(3)
d_group.to_edge(RIGHT)
weight_grid = digit.copy()
weight_grid.set_fill(BLACK, 0.5)
self.get_pixels_to_detect(weight_grid).set_fill(
GREEN, 0.5
)
self.get_surrounding_pixels_for_edge(weight_grid).set_fill(
RED, 0.5
)
weight_grid.move_to(digit)

edges = self.neuron.edges_in

self.make_edges_weighted(
edges, 1.5*np.random.random(len(edges)) - 0.5
)

Transform(

self.network_mob.layers[0],
self.network_mob.get_active_layer(0, np.random.random(16))
).update(1)

self.add(self.weighted_sum, digit, weight_grid)

self.digit = digit
self.weight_grid = weight_grid

def add_sigmoid_label(self):

name = TexText("Sigmoid")
sigma = self.weighted_sum[0][0]
name.next_to(sigma, UP)
name.to_edge(UP, SMALL_BUFF)

arrow = Arrow(

name.get_bottom(), sigma.get_top(),
buff = SMALL_BUFF,
use_rectangular_stem = False,
max_tip_length_to_length_ratio = 0.3
)

self.play(

Write(name),
ShowCreation(arrow),
)
self.sigmoid_name = name
self.sigmoid_arrow = arrow

def words_on_activation(self):

neuron = self.neuron
weighted_sum = self.weighted_sum

activation_word = TexText("Activation")

activation_word.next_to(neuron, RIGHT)
arrow = Arrow(neuron, weighted_sum.get_bottom())
arrow.set_color(WHITE)
words = TexText("How positive is this?")
words.next_to(self.weighted_sum, UP, SMALL_BUFF)

self.play(

FadeIn(activation_word),
neuron.set_fill, WHITE, 0.8,
)
self.wait()
self.play(
GrowArrow(arrow),
ReplacementTransform(activation_word, words),
)
self.wait(2)
self.play(FadeOut(arrow))

self.how_positive_words = words

def comment_on_need_for_bias(self):

neuron = self.neuron
weight_grid = self.weight_grid
colored_pixels = VGroup(
self.get_pixels_to_detect(weight_grid),
self.get_surrounding_pixels_for_edge(weight_grid),
)

words = TexText(

"Only activate meaningfully \\\\ when",
"weighted sum", "$> 10$"
)
words.set_color_by_tex("weighted", GREEN)
words.next_to(neuron, RIGHT)

self.play(Write(words, run_time = 2))

self.play(ApplyMethod(
colored_pixels.shift, MED_LARGE_BUFF*UP,
rate_func = there_and_back,
run_time = 2,
lag_ratio = 0.5
))
self.wait()

self.gt_ten = words[-1]

def add_bias(self):

bias = Tex("-10")
wn, rp = self.weighted_sum[-2:]
bias.next_to(wn, RIGHT, SMALL_BUFF)
bias.shift(0.02*UP)
rp.generate_target()
rp.target.next_to(bias, RIGHT, SMALL_BUFF)

rect = SurroundingRectangle(bias, buff = 0.5*SMALL_BUFF)

name = TexText("``bias''")
name.next_to(rect, DOWN)
VGroup(rect, name).set_color(BLUE)

self.play(

ReplacementTransform(
self.gt_ten.copy(), bias,
run_time = 2
),
MoveToTarget(rp),
)
self.wait(2)
self.play(
ShowCreation(rect),
Write(name)
)
self.wait(2)

self.bias_name = name

def summarize_weights_and_biases(self):

weight_grid = self.weight_grid
bias_name = self.bias_name

self.play(LaggedStartMap(

ApplyMethod, weight_grid,
lambda p : (p.set_fill,
random.choice([GREEN, GREEN, RED]),
random.random()
),
rate_func = there_and_back,
lag_ratio = 0.4,
run_time = 4
))
self.wait()
self.play(Indicate(bias_name))
self.wait(2)

###

def get_weighted_sum(self):
args = ["\\sigma \\big("]
for d in range(1, 4):
args += ["w_%d"%d, "a_%d"%d, "+"]
args += ["\\cdots", "+", "w_n", "a_n"]
args += ["\\big)"]
weighted_sum = Tex(*args)
weighted_sum.set_color_by_tex("w_", GREEN)
weighted_sum.set_color_by_tex("\\big", YELLOW)
weighted_sum.to_edge(UP, LARGE_BUFF)
weighted_sum.shift(RIGHT)

return weighted_sum

class BiasForInactiviyWords(Scene):

def construct(self):
words = TexText("Bias for inactivity")
words.set_color(BLUE)
words.set_width(FRAME_WIDTH - 1)
words.to_edge(UP)

self.play(Write(words))

self.wait(3)

class ContinualEdgeUpdate(VGroup):

CONFIG = {
"max_stroke_width" : 3,
"stroke_width_exp" : 7,
"n_cycles" : 5,
"colors" : [GREEN, GREEN, GREEN, RED],
}
def __init__(self, network_mob, **kwargs):
VGroup.__init__(self, **kwargs)
self.internal_time = 0
n_cycles = self.n_cycles
edges = VGroup(*it.chain(*network_mob.edge_groups))
self.move_to_targets = []
for edge in edges:
edge.colors = [
random.choice(self.colors)
for x in range(n_cycles)
]
msw = self.max_stroke_width
edge.widths = [
msw*random.random()**self.stroke_width_exp
for x in range(n_cycles)
]
edge.cycle_time = 1 + random.random()

edge.generate_target()

edge.target.set_stroke(edge.colors[0], edge.widths[0])
edge.become(edge.target)
self.move_to_targets.append(edge)
self.edges = edges
self.add(edges)
self.add_updater(lambda m, dt: self.update_edges(dt))

def update_edges(self, dt):

self.internal_time += dt
if self.internal_time < 1:
alpha = smooth(self.internal_time)
for move_to_target in self.move_to_targets:
move_to_target.update(alpha)
return
for edge in self.edges:
t = (self.internal_time-1)/edge.cycle_time
alpha = ((self.internal_time-1)%edge.cycle_time)/edge.cycle_time
low_n = int(t)%len(edge.colors)
high_n = int(t+1)%len(edge.colors)
color = interpolate_color(edge.colors[low_n], edge.colors[high_n], alpha)
width = interpolate(edge.widths[low_n], edge.widths[high_n], alpha)
edge.set_stroke(color, width)

class ShowRemainingNetwork(IntroduceWeights):

def construct(self):
self.force_skipping()
self.zoom_in_on_one_neuron()
self.revert_to_original_skipping_status()

self.show_all_of_second_layer()

self.count_in_biases()
self.compute_layer_two_of_weights_and_biases_count()
self.show_remaining_layers()
self.show_final_number()
self.tweak_weights()

def show_all_of_second_layer(self):

example_neuron = self.neuron
layer = self.network_mob.layers[1]

neurons = VGroup(*layer.neurons)

neurons.remove(example_neuron)

words = TexText("784", "weights", "per neuron")

words.next_to(layer.neurons[0], RIGHT)
words.to_edge(UP)

self.play(FadeIn(words))

last_edges = None
for neuron in neurons[:7]:
edges = neuron.edges_in
added_anims = []
if last_edges is not None:
added_anims += [
last_edges.set_stroke, None, 1
]
edges.set_stroke(width = 2)
self.play(
ShowCreation(edges, lag_ratio = 0.5),
FadeIn(neuron),
*added_anims,
run_time = 1.5
)
last_edges = edges
self.play(
LaggedStartMap(
ShowCreation, VGroup(*[
n.edges_in for n in neurons[7:]
]),
run_time = 3,
),
LaggedStartMap(
FadeIn, VGroup(*neurons[7:]),
run_time = 3,
),
VGroup(*last_edges[1:]).set_stroke, None, 1
)
self.wait()

self.weights_words = words

def count_in_biases(self):

neurons = self.network_mob.layers[1].neurons
words = TexText("One", "bias","for each")
words.next_to(neurons, RIGHT, buff = 2)
arrows = VGroup(*[
Arrow(
words.get_left(),
neuron.get_center(),
color = BLUE
)
for neuron in neurons
])

self.play(

FadeIn(words),
LaggedStartMap(
GrowArrow, arrows,
run_time = 3,
lag_ratio = 0.3,
)
)
self.wait()

self.bias_words = words

self.bias_arrows = arrows

def compute_layer_two_of_weights_and_biases_count(self):

ww1, ww2, ww3 = weights_words = self.weights_words
bb1, bb2, bb3 = bias_words = self.bias_words
bias_arrows = self.bias_arrows

times_16 = Tex("\\times 16")

times_16.next_to(ww1, RIGHT, SMALL_BUFF)
ww2.generate_target()
ww2.target.next_to(times_16, RIGHT)

bias_count = TexText("16", "biases")

bias_count.next_to(ww2.target, RIGHT, LARGE_BUFF)

self.play(

Write(times_16),
MoveToTarget(ww2),
FadeOut(ww3)
)
self.wait()
self.play(
ReplacementTransform(times_16.copy(), bias_count[0]),
FadeOut(bb1),
ReplacementTransform(bb2, bias_count[1]),
FadeOut(bb3),
LaggedStartMap(FadeOut, bias_arrows)
)
self.wait()

self.weights_count = VGroup(ww1, times_16, ww2)

self.bias_count = bias_count

def show_remaining_layers(self):

weights_count = self.weights_count
bias_count = self.bias_count
for count in weights_count, bias_count:
count.generate_target()
count.prefix = VGroup(*count.target[:-1])

added_weights = Tex(

"+16\\!\\times\\! 16 + 16 \\!\\times\\! 10"
)
added_weights.to_corner(UP+RIGHT)
weights_count.prefix.next_to(added_weights, LEFT, SMALL_BUFF)
weights_count.target[-1].next_to(
VGroup(weights_count.prefix, added_weights),
DOWN
)

added_biases = Tex("+ 16 + 10")

group = VGroup(bias_count.prefix, added_biases)
group.arrange(RIGHT, SMALL_BUFF)
group.next_to(weights_count.target[-1], DOWN, LARGE_BUFF)
bias_count.target[-1].next_to(group, DOWN)

network_mob = self.network_mob

edges = VGroup(*it.chain(*network_mob.edge_groups[1:]))
neurons = VGroup(*it.chain(*[
layer.neurons for layer in network_mob.layers[2:]
]))

self.play(

MoveToTarget(weights_count),
MoveToTarget(bias_count),
Write(added_weights, run_time = 1),
Write(added_biases, run_time = 1),
LaggedStartMap(
ShowCreation, edges,
run_time = 4,
lag_ratio = 0.3,
),
LaggedStartMap(
FadeIn, neurons,
run_time = 4,
lag_ratio = 0.3,
)
)
self.wait(2)

weights_count.add(added_weights)

bias_count.add(added_biases)

def show_final_number(self):

group = VGroup(
self.weights_count,
self.bias_count,
)
group.generate_target()
group.target.scale(0.8)
rect = SurroundingRectangle(group.target, buff = MED_SMALL_BUFF)
num_mob = Tex("13{,}002")
num_mob.scale(1.5)
num_mob.next_to(rect, DOWN)

self.play(

ShowCreation(rect),
MoveToTarget(group),
)
self.play(Write(num_mob))
self.wait()

self.final_number = num_mob

def tweak_weights(self):

learning = TexText("Learning $\\rightarrow$")
finding_words = TexText(
"Finding the right \\\\ weights and biases"
)
group = VGroup(learning, finding_words)
group.arrange(RIGHT)
group.scale(0.8)
group.next_to(self.final_number, DOWN, MED_LARGE_BUFF)

self.add(ContinualEdgeUpdate(self.network_mob))

self.wait(5)
self.play(Write(group))
self.wait(10)

###

def get_edge_weight_wandering_anim(self, edges):
for edge in edges:
edge.generate_target()
edge.target.set_stroke(
color = random.choice([GREEN, GREEN, GREEN, RED]),
width = 3*random.random()**7
)
self.play(
LaggedStartMap(
MoveToTarget, edges,
lag_ratio = 0.6,
run_time = 2,
),
*added_anims
)

class ImagineSettingByHand(Scene):

def construct(self):
randy = Randolph()
randy.scale(0.7)
randy.to_corner(DOWN+LEFT)

bubble = randy.get_bubble()

network_mob = NetworkMobject(
Network(sizes = [8, 6, 6, 4]),
neuron_stroke_color = WHITE
)
network_mob.scale(0.7)
network_mob.move_to(bubble.get_bubble_center())
network_mob.shift(MED_SMALL_BUFF*RIGHT + SMALL_BUFF*(UP+RIGHT))

self.add(randy, bubble, network_mob)

self.add(ContinualEdgeUpdate(network_mob))
self.play(randy.change, "pondering")
self.wait()
self.play(Blink(randy))
self.wait()
self.play(randy.change, "horrified", network_mob)
self.play(Blink(randy))
self.wait(10)

class WhenTheNetworkFails(MoreHonestMNistNetworkPreview):

CONFIG = {
"network_mob_config" : {"layer_to_layer_buff" : 2}
}
def construct(self):
self.setup_network_mob()
self.black_box()
self.incorrect_classification()
self.ask_about_weights()

def setup_network_mob(self):

self.network_mob.scale(0.8)
self.network_mob.to_edge(DOWN)

def black_box(self):

network_mob = self.network_mob
layers = VGroup(*network_mob.layers[1:3])
box = SurroundingRectangle(
layers,
stroke_color = WHITE,
fill_color = BLACK,
fill_opacity = 0.8,
)
words = TexText("...rather than treating this as a black box")
words.next_to(box, UP, LARGE_BUFF)

self.play(

Write(words, run_time = 2),
DrawBorderThenFill(box)
)
self.wait()
self.play(*list(map(FadeOut, [words, box])))

def incorrect_classification(self):

network = self.network
training_data, validation_data, test_data = load_data_wrapper()
for in_vect, result in test_data[20:]:
network_answer = np.argmax(network.feedforward(in_vect))
if network_answer != result:
break
self.feed_in_image(in_vect)

wrong = TexText("Wrong!")

wrong.set_color(RED)
wrong.next_to(self.network_mob.layers[-1], UP+RIGHT)
self.play(Write(wrong, run_time = 1))

def ask_about_weights(self):

question = TexText(
"What weights are used here?\\\\",
"What are they doing?"
)
question.next_to(self.network_mob, UP)

self.add(ContinualEdgeUpdate(self.network_mob))

self.play(Write(question))
self.wait(10)
###

def reset_display(self, *args):

pass

class EvenWhenItWorks(TeacherStudentsScene):

def construct(self):
self.teacher_says(
"Even when it works,\\\\",
"dig into why."
)
self.change_student_modes(*["pondering"]*3)
self.wait(7)

class IntroduceWeightMatrix(NetworkScene):

CONFIG = {
"network_mob_config" : {
"neuron_stroke_color" : WHITE,
"neuron_fill_color" : WHITE,
"neuron_radius" : 0.35,
"layer_to_layer_buff" : 2,
},
"layer_sizes" : [8, 6],
}
def construct(self):
self.setup_network_mob()
self.show_weighted_sum()
self.organize_activations_into_column()
self.organize_weights_as_matrix()
self.show_meaning_of_matrix_row()
self.connect_weighted_sum_to_matrix_multiplication()
self.add_bias_vector()
self.apply_sigmoid()
self.write_clean_final_expression()

def setup_network_mob(self):

self.network_mob.to_edge(LEFT, buff = LARGE_BUFF)
self.network_mob.layers[1].neurons.shift(0.02*RIGHT)

def show_weighted_sum(self):

self.fade_many_neurons()
self.activate_first_layer()
self.show_first_neuron_weighted_sum()
self.add_bias()
self.add_sigmoid()
##

def fade_many_neurons(self):

anims = []
neurons = self.network_mob.layers[1].neurons
for neuron in neurons[1:]:
neuron.save_state()
neuron.edges_in.save_state()
anims += [
neuron.fade, 0.8,
neuron.set_fill, None, 0,
neuron.edges_in.fade, 0.8,
]
anims += [
Animation(neurons[0]),
Animation(neurons[0].edges_in),
]
self.play(*anims)

def activate_first_layer(self):

layer = self.network_mob.layers[0]
activations = 0.7*np.random.random(len(layer.neurons))
active_layer = self.network_mob.get_active_layer(0, activations)
a_labels = VGroup(*[
Tex("a^{(0)}_%d"%d)
for d in range(len(layer.neurons))
])
for label, neuron in zip(a_labels, layer.neurons):
label.scale(0.75)
label.move_to(neuron)

self.play(

Transform(layer, active_layer),
Write(a_labels, run_time = 2)
)

self.a_labels = a_labels

def show_first_neuron_weighted_sum(self):

neuron = self.network_mob.layers[1].neurons[0]
a_labels = VGroup(*self.a_labels[:2]).copy()
a_labels.generate_target()
w_labels = VGroup(*[
Tex("w_{0, %d}"%d)
for d in range(len(a_labels))
])
weighted_sum = VGroup()
symbols = VGroup()
for a_label, w_label in zip(a_labels.target, w_labels):
a_label.scale(1./0.75)
plus = Tex("+")
weighted_sum.add(w_label, a_label, plus)
symbols.add(plus)
weighted_sum.add(
Tex("\\cdots"),
Tex("+"),
Tex("w_{0, n}"),
Tex("a^{(0)}_n"),
)

weighted_sum.arrange(RIGHT)

a1_label = Tex("a^{(1)}_0")
a1_label.next_to(neuron, RIGHT)
equals = Tex("=").next_to(a1_label, RIGHT)
weighted_sum.next_to(equals, RIGHT)

symbols.add(*weighted_sum[-4:-2])

w_labels.add(weighted_sum[-2])
a_labels.add(self.a_labels[-1].copy())
a_labels.target.add(weighted_sum[-1])
a_labels.add(VGroup(*self.a_labels[2:-1]).copy())
a_labels.target.add(VectorizedPoint(weighted_sum[-4].get_center()))

VGroup(a1_label, equals, weighted_sum).scale(

0.75, about_point = a1_label.get_left()
)

w_labels.set_color(GREEN)

w_labels.shift(0.6*SMALL_BUFF*DOWN)
a_labels.target.shift(0.5*SMALL_BUFF*UP)

self.play(

Write(a1_label),
Write(equals),
neuron.set_fill, None, 0.3,
run_time = 1
)
self.play(MoveToTarget(a_labels, run_time = 1.5))
self.play(
Write(w_labels),
Write(symbols),
)

self.a1_label = a1_label

self.a1_equals = equals
self.w_labels = w_labels
self.a_labels_in_sum = a_labels
self.symbols = symbols
self.weighted_sum = VGroup(w_labels, a_labels, symbols)

def add_bias(self):

weighted_sum = self.weighted_sum
bias = Tex("+\\,", "b_0")
bias.scale(0.75)
bias.next_to(weighted_sum, RIGHT, SMALL_BUFF)
bias.shift(0.5*SMALL_BUFF*DOWN)
name = TexText("Bias")
name.scale(0.75)
name.next_to(bias, DOWN, MED_LARGE_BUFF)
arrow = Arrow(name, bias, buff = SMALL_BUFF)
VGroup(name, arrow, bias).set_color(BLUE)

self.play(

FadeIn(name),
FadeIn(bias),
GrowArrow(arrow),
)

self.weighted_sum.add(bias)

self.bias = bias

self.bias_name = VGroup(name, arrow)

def add_sigmoid(self):

weighted_sum = self.weighted_sum
weighted_sum.generate_target()
sigma, lp, rp = mob = Tex("\\sigma\\big(\\big)")
# mob.scale(0.75)
sigma.move_to(weighted_sum.get_left())
sigma.shift(0.5*SMALL_BUFF*(DOWN+RIGHT))
lp.next_to(sigma, RIGHT, SMALL_BUFF)
weighted_sum.target.next_to(lp, RIGHT, SMALL_BUFF)
rp.next_to(weighted_sum.target, RIGHT, SMALL_BUFF)

name = TexText("Sigmoid")

name.next_to(sigma, UP, MED_LARGE_BUFF)
arrow = Arrow(name, sigma, buff = SMALL_BUFF)
sigmoid_name = VGroup(name, arrow)
VGroup(sigmoid_name, mob).set_color(YELLOW)

self.play(

FadeIn(mob),
MoveToTarget(weighted_sum),
MaintainPositionRelativeTo(self.bias_name, self.bias),
)
self.play(FadeIn(sigmoid_name))

self.sigma = sigma

self.sigma_parens = VGroup(lp, rp)
self.sigmoid_name = sigmoid_name

##

def organize_activations_into_column(self):
a_labels = self.a_labels.copy()
a_labels.generate_target()
column = a_labels.target
a_labels_in_sum = self.a_labels_in_sum

dots = Tex("\\vdots")

mid_as = VGroup(*column[2:-1])
Transform(mid_as, dots).update(1)
last_a = column[-1]
new_last_a = Tex(
last_a.get_tex().replace("7", "n")
)
new_last_a.replace(last_a)
Transform(last_a, new_last_a).update(1)

VGroup(
*column[:2] + [mid_as] + [column[-1]]

).arrange(DOWN)
column.shift(DOWN + 3.5*RIGHT)

pre_brackets = self.get_brackets(a_labels)

post_bracketes = self.get_brackets(column)
pre_brackets.set_fill(opacity = 0)

self.play(FocusOn(self.a_labels[0]))

self.play(LaggedStartMap(
Indicate, self.a_labels,
rate_func = there_and_back,
run_time = 1
))
self.play(
MoveToTarget(a_labels),
Transform(pre_brackets, post_bracketes),
run_time = 2
)
self.wait()
self.play(*[
LaggedStartMap(Indicate, mob, rate_func = there_and_back)
for mob in (a_labels, a_labels_in_sum)
])
self.wait()

self.a_column = a_labels

self.a_column_brackets = pre_brackets

def organize_weights_as_matrix(self):

a_column = self.a_column
a_column_brackets = self.a_column_brackets
w_brackets = a_column_brackets.copy()
w_brackets.next_to(a_column_brackets, LEFT, SMALL_BUFF)
lwb, rwb = w_brackets

w_labels = self.w_labels.copy()

w_labels.submobjects.insert(
2, self.symbols[-2].copy()
)
w_labels.generate_target()
w_labels.target.arrange(RIGHT)
w_labels.target.next_to(a_column[0], LEFT, buff = 0.8)
lwb.next_to(w_labels.target, LEFT, SMALL_BUFF)
lwb.align_to(rwb, UP)

row_1, row_k = [

VGroup(*list(map(Tex, [
"w_{%s, 0}"%i,
"w_{%s, 1}"%i,
"\\cdots",
"w_{%s, n}"%i,
])))
for i in ("1", "k")
]
dots_row = VGroup(*list(map(Tex, [
"\\vdots", "\\vdots", "\\ddots", "\\vdots"
])))

lower_rows = VGroup(row_1, dots_row, row_k)

lower_rows.scale(0.75)
last_row = w_labels.target
for row in lower_rows:
for target, mover in zip(last_row, row):
mover.move_to(target)
if "w" in mover.get_tex():
mover.set_color(GREEN)
row.next_to(last_row, DOWN, buff = 0.45)
last_row = row

self.play(

MoveToTarget(w_labels),
Write(w_brackets, run_time = 1)
)
self.play(FadeIn(
lower_rows,
run_time = 3,
lag_ratio = 0.5,
))
self.wait()

self.top_matrix_row = w_labels

self.lower_matrix_rows = lower_rows
self.matrix_brackets = w_brackets

def show_meaning_of_matrix_row(self):

row = self.top_matrix_row
edges = self.network_mob.layers[1].neurons[0].edges_in.copy()
edges.set_stroke(GREEN, 5)
rect = SurroundingRectangle(row, color = GREEN_B)

self.play(ShowCreation(rect))

for x in range(2):
self.play(LaggedStartMap(
ShowCreationThenDestruction, edges,
lag_ratio = 0.8
))
self.wait()

self.top_row_rect = rect

def connect_weighted_sum_to_matrix_multiplication(self):

a_column = self.a_column
a_brackets = self.a_column_brackets
top_row_rect = self.top_row_rect

column_rect = SurroundingRectangle(a_column)

equals = Tex("=")

equals.next_to(a_brackets, RIGHT)
result_brackets = a_brackets.copy()
result_terms = VGroup()
for i in 0, 1, 4, -1:
a = a_column[i]
if i == 4:
mob = Tex("\\vdots")
else:
# mob = Circle(radius = 0.2, color = YELLOW)
mob = Tex("?").scale(1.3).set_color(YELLOW)
result_terms.add(mob.move_to(a))
VGroup(result_brackets, result_terms).next_to(equals, RIGHT)

brace = Brace(

VGroup(self.w_labels, self.a_labels_in_sum), DOWN
)
arrow = Arrow(
brace.get_bottom(),
result_terms[0].get_top(),
buff = SMALL_BUFF
)

self.play(

GrowArrow(arrow),
GrowFromCenter(brace),
)
self.play(
Write(equals),
FadeIn(result_brackets),
)
self.play(ShowCreation(column_rect))
self.play(ReplacementTransform(
VGroup(top_row_rect, column_rect).copy(),
result_terms[0]
))
self.play(LaggedStartMap(
FadeIn, VGroup(*result_terms[1:])
))
self.wait(2)
self.show_meaning_of_lower_rows(
arrow, brace, top_row_rect, result_terms
)
self.play(*list(map(FadeOut, [
result_terms, result_brackets, equals, column_rect
])))

def show_meaning_of_lower_rows(self, arrow, brace, row_rect, result_terms):

n1, n2, nk = neurons = VGroup(*[
self.network_mob.layers[1].neurons[i]
for i in (0, 1, -1)
])
for n in neurons:
n.save_state()
n.edges_in.save_state()

rect2 = SurroundingRectangle(result_terms[1])

rectk = SurroundingRectangle(result_terms[-1])
VGroup(rect2, rectk).set_color(WHITE)
row2 = self.lower_matrix_rows[0]
rowk = self.lower_matrix_rows[-1]

def show_edges(neuron):

self.play(LaggedStartMap(
ShowCreationThenDestruction,
neuron.edges_in.copy().set_stroke(GREEN, 5),
lag_ratio = 0.7,
run_time = 1,
))

self.play(

row_rect.move_to, row2,
n1.fade,
n1.set_fill, None, 0,
n1.edges_in.set_stroke, None, 1,
n2.set_stroke, WHITE, 3,
n2.edges_in.set_stroke, None, 3,
ReplacementTransform(arrow, rect2),
FadeOut(brace),
)
show_edges(n2)
self.play(
row_rect.move_to, rowk,
n2.restore,
n2.edges_in.restore,
nk.set_stroke, WHITE, 3,
nk.edges_in.set_stroke, None, 3,
ReplacementTransform(rect2, rectk),
)
show_edges(nk)
self.play(
n1.restore,
n1.edges_in.restore,
nk.restore,
nk.edges_in.restore,
FadeOut(rectk),
FadeOut(row_rect),
)

def add_bias_vector(self):

bias = self.bias
bias_name = self.bias_name
a_column_brackets = self.a_column_brackets
a_column = self.a_column

plus = Tex("+")

b_brackets = a_column_brackets.copy()
b_column = VGroup(*list(map(Tex, [
"b_0", "b_1", "\\vdots", "b_n",
])))
b_column.scale(0.85)
b_column.arrange(DOWN, buff = 0.35)
b_column.move_to(a_column)
b_column.set_color(BLUE)
plus.next_to(a_column_brackets, RIGHT)
VGroup(b_brackets, b_column).next_to(plus, RIGHT)

bias_rect = SurroundingRectangle(bias)

self.play(ShowCreation(bias_rect))

self.play(FadeOut(bias_rect))
self.play(
Write(plus),
Write(b_brackets),
Transform(self.bias[1].copy(), b_column[0]),
run_time = 1
)
self.play(LaggedStartMap(
FadeIn, VGroup(*b_column[1:])
))
self.wait()

self.bias_plus = plus

self.b_brackets = b_brackets
self.b_column = b_column

def apply_sigmoid(self):

expression_bounds = VGroup(
self.matrix_brackets[0], self.b_brackets[1]
)
sigma = self.sigma.copy()
slp, srp = self.sigma_parens.copy()

big_lp, big_rp = parens = Tex("()")

parens.scale(3)
parens.stretch_to_fit_height(expression_bounds.get_height())
big_lp.next_to(expression_bounds, LEFT, SMALL_BUFF)
big_rp.next_to(expression_bounds, RIGHT, SMALL_BUFF)
parens.set_color(YELLOW)

self.play(

sigma.scale, 2,
sigma.next_to, big_lp, LEFT, SMALL_BUFF,
Transform(slp, big_lp),
Transform(srp, big_rp),
)
self.wait(2)

self.big_sigma_group = VGroup(VGroup(sigma), slp, srp)

def write_clean_final_expression(self):

self.fade_weighted_sum()
expression = Tex(
"\\textbf{a}^{(1)}",
"=",
"\\sigma",
"\\big(",
"\\textbf{W}",
"\\textbf{a}^{(0)}",
"+",
"\\textbf{b}",
"\\big)",
)
expression.set_color_by_tex_to_color_map({
"sigma" : YELLOW,
"big" : YELLOW,
"W" : GREEN,
"\\textbf{b}" : BLUE
})
expression.next_to(self.big_sigma_group, UP, LARGE_BUFF)
a1, equals, sigma, lp, W, a0, plus, b, rp = expression

neuron_anims = []

neurons = VGroup(*self.network_mob.layers[1].neurons[1:])
for neuron in neurons:
neuron_anims += [
neuron.restore,
neuron.set_fill, None, random.random()
]
neuron_anims += [
neuron.edges_in.restore
]
neurons.add_to_back(self.network_mob.layers[1].neurons[0])

self.play(ReplacementTransform(

VGroup(
self.top_matrix_row, self.lower_matrix_rows,
self.matrix_brackets
).copy(),
VGroup(W),
))
self.play(ReplacementTransform(
VGroup(self.a_column, self.a_column_brackets).copy(),
VGroup(VGroup(a0)),
))
self.play(
ReplacementTransform(
VGroup(self.b_column, self.b_brackets).copy(),
VGroup(VGroup(b))
),
ReplacementTransform(
self.bias_plus.copy(), plus
)
)
self.play(ReplacementTransform(
self.big_sigma_group.copy(),
VGroup(sigma, lp, rp)
))
self.wait()
self.play(*neuron_anims, run_time = 2)
self.play(
ReplacementTransform(neurons.copy(), a1),
FadeIn(equals)
)
self.wait(2)

def fade_weighted_sum(self):

self.play(*list(map(FadeOut, [
self.a1_label, self.a1_equals,
self.sigma, self.sigma_parens,
self.weighted_sum,
self.bias_name,
self.sigmoid_name,
])))
###

def get_brackets(self, mob):

lb, rb = both = Tex("\\big[\\big]")
both.set_width(mob.get_width())
both.stretch_to_fit_height(1.2*mob.get_height())
lb.next_to(mob, LEFT, SMALL_BUFF)
rb.next_to(mob, RIGHT, SMALL_BUFF)
return both

class HorrifiedMorty(Scene):

def construct(self):
morty = Mortimer()
morty.flip()
morty.scale(2)

for mode in "horrified", "hesitant":

self.play(
morty.change, mode,
morty.look, UP,
)
self.play(Blink(morty))
self.wait(2)

class SigmoidAppliedToVector(Scene):

def construct(self):
tex = Tex("""
\\sigma \\left(
\\left[\\begin{array}{c}
x \\\\ y \\\\ z
\\end{array}\\right]
\\right) =
\\left[\\begin{array}{c}
\\sigma(x) \\\\ \\sigma(y) \\\\ \\sigma(z)
\\end{array}\\right]
""")
tex.set_width(FRAME_WIDTH - 1)
tex.to_edge(DOWN)
indices = it.chain(
[0], list(range(1, 5)), list(range(16, 16+4)),
list(range(25, 25+2)), [25+3],
list(range(29, 29+2)), [29+3],
list(range(33, 33+2)), [33+3],
)
for i in indices:
tex[i].set_color(YELLOW)
self.add(tex)
self.wait()

class EoLA3Wrapper(PiCreatureScene):

def construct(self):
morty = self.pi_creature
rect = ScreenRectangle(height = 5)
rect.next_to(morty, UP+LEFT)
rect.to_edge(UP, buff = LARGE_BUFF)
title = TexText("Essence of linear algebra")
title.next_to(rect, UP)

self.play(

ShowCreation(rect),
FadeIn(title),
morty.change, "raise_right_hand", rect
)
self.wait(4)

class FeedForwardCode(ExternallyAnimatedScene):

pass

class NeuronIsFunction(MoreHonestMNistNetworkPreview):

CONFIG = {
"network_mob_config" : {
"layer_to_layer_buff" : 2
}
}
def construct(self):
self.setup_network_mob()
self.activate_network()
self.write_neuron_holds_a_number()
self.feed_in_new_image(8, 7)
self.neuron_is_function()
self.show_neuron_as_function()
self.fade_network_back_in()
self.network_is_a_function()
self.feed_in_new_image(9, 4)
self.wait(2)
def setup_network_mob(self):
self.network_mob.scale(0.7)
self.network_mob.to_edge(DOWN)
self.network_mob.shift(LEFT)

def activate_network(self):

network_mob = self.network_mob
self.image_map = get_organized_images()
in_vect = self.image_map[3][0]
mnist_mob = MNistMobject(in_vect)
mnist_mob.next_to(network_mob, LEFT, MED_LARGE_BUFF, UP)
activations = self.network.get_activation_of_all_layers(in_vect)
for i, activation in enumerate(activations):
layer = self.network_mob.layers[i]
Transform(
layer, self.network_mob.get_active_layer(i, activation)
).update(1)
self.add(mnist_mob)

self.image_rect, self.curr_image = mnist_mob

def write_neuron_holds_a_number(self):

neuron_word = TexText("Neuron")
arrow = Arrow(ORIGIN, DOWN, color = BLUE)
thing_words = TexText("Thing that holds \\\\ a number")
group = VGroup(neuron_word, arrow, thing_words)
group.arrange(DOWN)
group.to_corner(UP+RIGHT, buff = LARGE_BUFF)

neuron = self.network_mob.layers[2].neurons[2]

decimal = DecimalNumber(neuron.get_fill_opacity())
decimal.set_width(0.7*neuron.get_width())
decimal.move_to(neuron)
neuron_group = VGroup(neuron, decimal)
neuron_group.save_state()
decimal.set_fill(opacity = 0)

self.play(

neuron_group.restore,
neuron_group.scale, 3,
neuron_group.next_to, neuron_word, LEFT,
FadeIn(neuron_word),
GrowArrow(arrow),
FadeIn(
thing_words, run_time = 2,
rate_func = squish_rate_func(smooth, 0.3, 1)
)
)
self.wait()
self.play(neuron_group.restore)

self.neuron_word = neuron_word

self.neuron_word_arrow = arrow
self.thing_words = thing_words
self.neuron = neuron
self.decimal = decimal

def feed_in_new_image(self, digit, choice):

in_vect = self.image_map[digit][choice]

args = []

for s in "answer_rect", "curr_image", "image_rect":
if hasattr(self, s):
args.append(getattr(self, s))
else:
args.append(VectorizedPoint())
MoreHonestMNistNetworkPreview.reset_display(self, *args)
self.feed_in_image(in_vect)

def neuron_is_function(self):

thing_words = self.thing_words
cross = Cross(thing_words)
function_word = TexText("Function")
function_word.move_to(thing_words, UP)

self.play(

thing_words.fade,
ShowCreation(cross)
)
self.play(
FadeIn(function_word),
VGroup(thing_words, cross).to_edge, DOWN,
)
self.wait()

self.function_word = function_word

def show_neuron_as_function(self):

neuron = self.neuron.copy()
edges = neuron.edges_in.copy()
prev_layer = self.network_mob.layers[1].copy()

arrow = Arrow(ORIGIN, RIGHT, color = BLUE)

arrow.next_to(neuron, RIGHT, SMALL_BUFF)
decimal = DecimalNumber(neuron.get_fill_opacity())
decimal.next_to(arrow, RIGHT)

self.play(

FadeOut(self.network_mob),
*list(map(Animation, [neuron, edges, prev_layer]))
)
self.play(LaggedStartMap(
ShowCreationThenDestruction,
edges.copy().set_stroke(YELLOW, 4),
))
self.play(
GrowArrow(arrow),
Transform(self.decimal, decimal)
)
self.wait(2)

self.non_faded_network_parts = VGroup(

neuron, edges, prev_layer
)
self.neuron_arrow = arrow

def fade_network_back_in(self):

anims = [
FadeIn(
mob,
run_time = 2,
lag_ratio = 0.5
)
for mob in (self.network_mob.layers, self.network_mob.edge_groups)
]
anims += [
FadeOut(self.neuron_arrow),
FadeOut(self.decimal),
]
anims.append(Animation(self.non_faded_network_parts))

self.play(*anims)

self.remove(self.non_faded_network_parts)

def network_is_a_function(self):

neuron_word = self.neuron_word
network_word = TexText("Network")
network_word.set_color(YELLOW)
network_word.move_to(neuron_word)

func_tex = Tex(

"f(a_0, \\dots, a_{783}) = ",
"""\\left[
\\begin{array}{c}
y_0 \\\\ \\vdots \\\\ y_{9}
\\end{array}
\\right]"""
)
func_tex.to_edge(UP)
func_tex.shift(MED_SMALL_BUFF*LEFT)

self.play(

ReplacementTransform(neuron_word, network_word),
FadeIn(func_tex)
)

###

def reset_display(self, answer_rect, image, image_rect):
#Don't do anything, just record these args
self.answer_rect = answer_rect
self.curr_image = image
self.image_rect = image_rect
return

class ComplicationIsReassuring(TeacherStudentsScene):

def construct(self):
self.student_says(
"It kind of has to \\\\ be complicated, right?",
target_mode = "speaking",
student_index = 0
)
self.play(self.teacher.change, "happy")
self.wait(4)

class NextVideo(MoreHonestMNistNetworkPreview, PiCreatureScene):

CONFIG = {
"network_mob_config" : {
"neuron_stroke_color" : WHITE,
"layer_to_layer_buff" : 2.5,
"brace_for_large_layers" : False,
}
}
def setup(self):
MoreHonestMNistNetworkPreview.setup(self)
PiCreatureScene.setup(self)

def construct(self):

self.network_and_data()
self.show_next_video()
self.talk_about_subscription()
self.show_video_neural_network()

def network_and_data(self):

morty = self.pi_creature
network_mob = self.network_mob
network_mob.to_edge(LEFT)
for obj in network_mob, self:
obj.remove(network_mob.output_labels)
network_mob.scale(0.7)
network_mob.shift(RIGHT)
edge_update = ContinualEdgeUpdate(network_mob)

training_data, validation_data, test_data = load_data_wrapper()

data_mobs = VGroup()
for vect, num in test_data[:30]:
image = MNistMobject(vect)
image.set_height(0.7)
arrow = Arrow(ORIGIN, RIGHT, color = BLUE)
num_mob = Tex(str(num))
group = Group(image, arrow, num_mob)
group.arrange(RIGHT, buff = SMALL_BUFF)
group.next_to(ORIGIN, RIGHT)
data_mobs.add(group)

data_mobs.next_to(network_mob, UP)

self.add(edge_update)

self.play(morty.change, "confused", network_mob)
self.wait(2)
for data_mob in data_mobs:
self.add(data_mob)
self.wait(0.2)
self.remove(data_mob)

self.content = network_mob

self.edge_update = edge_update

def show_next_video(self):

morty = self.pi_creature
content = self.content

video = VideoIcon()

video.set_height(3)
video.set_fill(RED, 0.8)
video.next_to(morty, UP+LEFT)

rect = SurroundingRectangle(video)

rect.set_stroke(width = 0)
rect.set_fill(BLACK, 0.5)

words = TexText("On learning")

words.next_to(video, UP)

if self.edge_update.internal_time < 1:

self.edge_update.internal_time = 2
self.play(
content.set_height, 0.8*video.get_height(),
content.move_to, video,
morty.change, "raise_right_hand",
FadeIn(rect),
FadeIn(video),
)
self.add_foreground_mobjects(rect, video)
self.wait(2)
self.play(Write(words))
self.wait(2)

self.video = Group(content, rect, video, words)

def talk_about_subscription(self):

morty = self.pi_creature
morty.generate_target()
morty.target.change("hooray")
morty.target.rotate(
np.pi, axis = UP, about_point = morty.get_left()
)
morty.target.shift(LEFT)
video = self.video
subscribe_word = TexText(
"Subscribe", "!",
arg_separator = ""
)
bang = subscribe_word[1]
subscribe_word.to_corner(DOWN+RIGHT)
subscribe_word.shift(3*UP)
q_mark = TexText("?")
q_mark.move_to(bang, LEFT)
arrow = Arrow(ORIGIN, DOWN, color = RED, buff = 0)
arrow.next_to(subscribe_word, DOWN)
arrow.shift(MED_LARGE_BUFF * RIGHT)

self.play(

Write(subscribe_word),
self.video.shift, 3*LEFT,
MoveToTarget(morty),
)
self.play(GrowArrow(arrow))
self.wait(2)
self.play(morty.change, "maybe", arrow)
self.play(Transform(bang, q_mark))
self.wait(3)

def show_video_neural_network(self):

morty = self.pi_creature

network_mob, rect, video, words = self.video

network_mob.generate_target(use_deepcopy = True)
network_mob.target.set_height(5)
network_mob.target.to_corner(UP+LEFT)
neurons = VGroup(*network_mob.target.layers[-1].neurons[:2])
neurons.set_stroke(width = 0)

video.generate_target()

video.target.set_fill(opacity = 1)
video.target.set_height(neurons.get_height())
video.target.move_to(neurons, LEFT)

self.play(

MoveToTarget(network_mob),
MoveToTarget(video),
FadeOut(words),
FadeOut(rect),
morty.change, "raise_left_hand"
)

neuron_pairs = VGroup(*[

VGroup(*network_mob.layers[-1].neurons[2*i:2*i+2])
for i in range(1, 5)
])
for pair in neuron_pairs:
video = video.copy()
video.move_to(pair, LEFT)
pair.target = video

self.play(LaggedStartMap(

MoveToTarget, neuron_pairs,
run_time = 3
))
self.play(morty.change, "shruggie")
self.wait(10)

###

class NNPatreonThanks(PatreonThanks):
CONFIG = {
"specific_patrons" : [
"Desmos",
"Burt Humburg",
"CrypticSwarm",
"Juan Benet",
"Ali Yahya",
"William",
"Mayank M. Mehrotra",
"Lukas Biewald",
"Samantha D. Suplee",
"Yana Chernobilsky",
"Kaustuv DeBiswas",
"Kathryn Schmiedicke",
"Yu Jun",
"Dave Nicponski",
"Damion Kistler",
"Markus Persson",
"Yoni Nazarathy",
"Ed Kellett",
"Joseph John Cox",
"Luc Ritchie",
"Andy Nichols",
"Harsev Singh",
"Mads Elvheim",
"Erik Sundell",
"Xueqi Li",
"David G. Stork",
"Tianyu Ge",
"Ted Suzman",
"Linh Tran",
"Andrew Busey",
"Michael McGuffin",
"John Haley",
"Ankalagon",
"Eric Lavault",
"Boris Veselinovich",
"Julian Pulgarin",
"Jeff Linse",
"Cooper Jones",
"Ryan Dahl",
"Mark Govea",
"Robert Teed",
"Jason Hise",
"Meshal Alshammari",
"Bernd Sing",
"James Thornton",
"Mustafa Mahdi",
"Mathew Bramson",
"Jerry Ling",
"Vecht",
"Shimin Kuang",
"Rish Kundalia",
"Achille Brighton",
"Ripta Pasay",
]
}

class PiCreatureGesture(PiCreatureScene):

def construct(self):
self.play(self.pi_creature.change, "raise_right_hand")
self.wait(5)
self.play(self.pi_creature.change, "happy")
self.wait(4)

class IntroduceReLU(IntroduceSigmoid):

CONFIG = {
"x_axis_label" : "$a$"
}
def construct(self):
self.setup_axes()
self.add_title()
self.add_graph()
self.old_school()
self.show_ReLU()
self.label_input_regions()

def old_school(self):

sigmoid_graph = self.sigmoid_graph
sigmoid_title = VGroup(
self.sigmoid_name,
self.equation
)
cross = Cross(sigmoid_title)
old_school = TexText("Old school")
old_school.to_corner(UP+RIGHT)
old_school.set_color(RED)
arrow = Arrow(
old_school.get_bottom(),
self.equation.get_right(),
color = RED
)

self.play(ShowCreation(cross))

self.play(
Write(old_school, run_time = 1),
GrowArrow(arrow)
)
self.wait(2)
self.play(
ApplyMethod(
VGroup(cross, sigmoid_title).shift,
FRAME_X_RADIUS*RIGHT,
rate_func = running_start
),
FadeOut(old_school),
FadeOut(arrow),
)
self.play(ShowCreation(
self.sigmoid_graph,
rate_func = lambda t : smooth(1-t),
remover = True
))

def show_ReLU(self):

graph = VGroup(
Line(
self.coords_to_point(-7, 0),
self.coords_to_point(0, 0),
),
Line(
self.coords_to_point(0, 0),
self.coords_to_point(4, 4),
),
)
graph.set_color(YELLOW)
char = self.x_axis_label.replace("$", "")
equation = TexText("ReLU($%s$) = max$(0, %s)$"%(char, char))
equation.shift(FRAME_X_RADIUS*LEFT/2)
equation.to_edge(UP)
equation.add_background_rectangle()
name = TexText("Rectified linear unit")
name.move_to(equation)
name.add_background_rectangle()

self.play(Write(equation))

self.play(ShowCreation(graph), Animation(equation))
self.wait(2)
self.play(
Write(name),
equation.shift, DOWN
)
self.wait(2)

self.ReLU_graph = graph

def label_input_regions(self):

l1, l2 = self.ReLU_graph
neg_words = TexText("Inactive")
neg_words.set_color(RED)
neg_words.next_to(self.coords_to_point(-2, 0), UP)

pos_words = TexText("Same as $f(a) = a$")

pos_words.set_color(GREEN)
pos_words.next_to(
self.coords_to_point(1, 1),
DOWN+RIGHT
)

self.revert_to_original_skipping_status()

self.play(ShowCreation(l1.copy().set_color(RED)))
self.play(Write(neg_words))
self.wait()
self.play(ShowCreation(l2.copy().set_color(GREEN)))
self.play(Write(pos_words))
self.wait(2)

class CompareSigmoidReLUOnDeepNetworks(PiCreatureScene):

def construct(self):
morty, lisha = self.morty, self.lisha
sigmoid_graph = FunctionGraph(
sigmoid,
x_min = -5,
x_max = 5,
)
sigmoid_graph.stretch_to_fit_width(3)
sigmoid_graph.set_color(YELLOW)
sigmoid_graph.next_to(lisha, UP+LEFT)
sigmoid_graph.shift_onto_screen()
sigmoid_name = TexText("Sigmoid")
sigmoid_name.next_to(sigmoid_graph, UP)
sigmoid_graph.add(sigmoid_name)

slow_learner = TexText("Slow learner")

slow_learner.set_color(YELLOW)
slow_learner.to_corner(UP+LEFT)
slow_arrow = Arrow(
slow_learner.get_bottom(),
sigmoid_graph.get_top(),
)

relu_graph = VGroup(

Line(2*LEFT, ORIGIN),
Line(ORIGIN, np.sqrt(2)*(RIGHT+UP)),
)
relu_graph.set_color(BLUE)
relu_graph.next_to(lisha, UP+RIGHT)
relu_name = TexText("ReLU")
relu_name.move_to(relu_graph, UP)
relu_graph.add(relu_name)

network_mob = NetworkMobject(Network(

sizes = [6, 4, 5, 4, 3, 5, 2]
))
network_mob.scale(0.8)
network_mob.to_edge(UP, buff = MED_SMALL_BUFF)
network_mob.shift(RIGHT)
edge_update = ContinualEdgeUpdate(
network_mob, stroke_width_exp = 1,
)

self.play(

FadeIn(sigmoid_name),
ShowCreation(sigmoid_graph),
lisha.change, "raise_left_hand",
morty.change, "pondering"
)
self.play(
Write(slow_learner, run_time = 1),
GrowArrow(slow_arrow)
)
self.wait()
self.play(
FadeIn(relu_name),
ShowCreation(relu_graph),
lisha.change, "raise_right_hand",
morty.change, "thinking"
)
self.play(FadeIn(network_mob))
self.add(edge_update)
self.wait(10)

###
def create_pi_creatures(self):

morty = Mortimer()
morty.shift(FRAME_X_RADIUS*RIGHT/2).to_edge(DOWN)
lisha = PiCreature(color = BLUE_C)
lisha.shift(FRAME_X_RADIUS*LEFT/2).to_edge(DOWN)
self.morty, self.lisha = morty, lisha
return morty, lisha

class ShowAmplify(PiCreatureScene):

def construct(self):
morty = self.pi_creature
rect = ScreenRectangle(height = 5)
rect.to_corner(UP+LEFT)
rect.shift(DOWN)
email = TexText("[e-mail]")
email.next_to(rect, UP)

self.play(

ShowCreation(rect),
morty.change, "raise_right_hand"
)
self.wait(2)
self.play(Write(email))
self.play(morty.change, "happy", rect)
self.wait(10)

class Thumbnail(NetworkScene):

CONFIG = {
"network_mob_config" : {
'neuron_stroke_color' : WHITE,
'layer_to_layer_buff': 1.25,
},
}
def construct(self):
network_mob = self.network_mob
network_mob.set_height(FRAME_HEIGHT - 1)
for layer in network_mob.layers:
layer.neurons.set_stroke(width = 5)

network_mob.set_height(5)

network_mob.to_edge(DOWN)
network_mob.to_edge(LEFT, buff=1)

subtitle = TexText(

"From the\\\\",
"ground up\\\\",
)
# subtitle.arrange(
# DOWN,
# buff=0.25,
# aligned_edge=LEFT,
# )
subtitle.set_color(YELLOW)
subtitle.set_height(2.75)
subtitle.next_to(network_mob, RIGHT, buff=MED_LARGE_BUFF)

edge_update = ContinualEdgeUpdate(

network_mob,
max_stroke_width = 10,
stroke_width_exp = 4,
)
edge_update.internal_time = 3
edge_update.update(0)

for mob in network_mob.family_members_with_points():

if mob.get_stroke_width() < 2:
mob.set_stroke(width=2)
title = TexText("Neural Networks")
title.scale(3)
title.to_edge(UP)

self.add(network_mob)

self.add(subtitle)
self.add(title)

# Extra

class NeuralNetImageAgain(Scene):
def construct(self):
layers = VGroup()
for length in [16, 16, 16, 10]:
circs = VGroup(*[
Circle(radius=1)
for x in range(length)
])
circs.arrange(DOWN, buff=0.5)
circs.set_stroke(WHITE, 2)
layers.add(circs)
layers.set_height(6.5)
layers.arrange(RIGHT, buff=2.5)

dots = Tex("\\vdots")

dots.move_to(layers[0])
layers[0][:8].next_to(dots, UP, MED_SMALL_BUFF)
layers[0][8:].next_to(dots, DOWN, MED_SMALL_BUFF)

for layer in layers[1:3]:

for node in layer:
node.set_fill(WHITE, opacity=random.random())
layers[3][6].set_fill(WHITE, 0.9)

all_edges = VGroup()

for l1, l2 in zip(layers, layers[1:]):
edges = VGroup()
for n1, n2 in it.product(l1, l2):
edge = Line(
n1.get_center(), n2.get_center(),
buff=n1.get_height() / 2
)
edge.set_stroke(WHITE, 1, opacity=0.75)
# edge.set_stroke(
# color=random.choice([BLUE, RED]),
# width=3 * random.random()**6,
# # opacity=0.5
# )
edges.add(edge)
all_edges.add(edges)

network = VGroup(all_edges, layers, dots)

brace = Brace(network, LEFT)

self.add(network)

self.add(brace)