JavaFX adding KeyFrames in KeyHandlers

So I am currently trying to create a project where you can move a box. The Box movements get "recorded" via KeyFrames, and by confirmation via SpaceBar you can play the movements again.
My Problem is, that upon pressing the SpaceBar, the Box just snaps to its starting point (100,100) for a split second and then goes back to whatever position it was before.

Here's my code:

import javafx.animation.KeyFrame;
import javafx.animation.Timeline;
import javafx.application.Application;
import javafx.scene.Group;
import javafx.scene.Scene;
import javafx.scene.input.KeyCode;
import javafx.scene.paint.Color;
import javafx.scene.shape.Rectangle;
import javafx.stage.Stage;
import javafx.util.Duration;

public class Main extends Application {
    static Rectangle rectangle;
    Timeline timeline = new Timeline();
    KeyFrame keyFrame;
    @Override
    public void start(Stage primaryStage) throws Exception{

        initializeRectangle();
        Group group = new Group();
        group.getChildren().add(rectangle);

        Scene scene = new Scene(group, 300, 300);
        addKeyHandlers(scene);

        primaryStage.setScene(scene);
        primaryStage.show();
    }
    private void initializeRectangle(){
        rectangle = new Rectangle();
        rectangle.setX(100);
        rectangle.setY(100);
        rectangle.setHeight(50);
        rectangle.setWidth(50);
        rectangle.setFill(Color.BLACK);
    }

    private void addKeyHandlers(Scene scene){

        scene.setOnKeyPressed(e -> {
            if (e.getCode() == KeyCode.S) {
                rectangle.setY(rectangle.getY() + 3);
                keyFrame = new KeyFrame(new Duration(20), actionEvent -> {
                    rectangle.setY(rectangle.getY() + 3);
                });
                timeline.getKeyFrames().addAll(keyFrame);
            }
            if (e.getCode() == KeyCode.W) {
                rectangle.setY(rectangle.getY() - 3);
                keyFrame = new KeyFrame(new Duration(20), actionEvent -> {
                    rectangle.setY(rectangle.getY() - 3);
                });
                timeline.getKeyFrames().addAll(keyFrame);
            }
            if (e.getCode() == KeyCode.D) {
                rectangle.setX(rectangle.getX() + 3);
                keyFrame = new KeyFrame(new Duration(20), actionEvent -> {
                    rectangle.setX(rectangle.getX() + 3);
                });
                timeline.getKeyFrames().addAll(keyFrame);
            }
            if (e.getCode() == KeyCode.A) {
                rectangle.setX(rectangle.getX() - 3);
                keyFrame = new KeyFrame(new Duration(20), actionEvent -> {
                    rectangle.setX(rectangle.getX() - 3);
                });
                timeline.getKeyFrames().addAll(keyFrame);
            }
            if(e.getCode() == KeyCode.SPACE){
                rectangle.setX(100);
                rectangle.setY(100);
                timeline.play();
            }
        });
    }


    public static void main(String[] args) {
        launch(args);
    }
}

Note: I am an absolute beginner in Java/JavaFX and I couldn't find any solution to this problem on the internet, and my professor isn't responding me (oof)