Sliding Blocks Won't Slide

Hello, my name is Luis,
I have been trying to make a Tetris/CandyCrush like mechanic for a small game where players shoot blocks to acquire points. Basically there is a wall of blocks and if you shoot a block it will destroy any neighbors of the same color in a cascade effect awarding more points. The idea is for the blocks on top of it to slide down the Z axis when blocks under it are destroyed. But they sometimes just float there as long as there is contact with any other blocks. I tried overriding the physics material and setting the friction to zero but that only seems to affect the player, not the other actors in the scene.
The only solution I can think of right now is giving a small space between blocks, but i would like to avoid that.

Here are some screenshots and the c++ class code to demonstrate the problem:
(Problems present themselves the same way in both the server and the clients)

// Fill out your copyright notice in the Description page of Project Settings.

#include "ColorBlock.h"
#include "LuisMFinalCharacter.h"
#include "Components/StaticMeshComponent.h"
#include "Components/BoxComponent.h"
#include "Components/SphereComponent.h"
#include "PhysicalMaterials/PhysicalMaterial.h"
#include "Materials/MaterialInstanceDynamic.h"
#include "Net/UnrealNetwork.h"

// Sets default values
AColorBlock::AColorBlock()
{
 	// Set this actor to call Tick() every frame.  You can turn this off to improve performance if you don't need it.
	PrimaryActorTick.bCanEverTick = true;

    m_Bounds = CreateDefaultSubobject<UBoxComponent>("Bounds");
    m_Bounds->SetIsReplicated(true);
    m_Bounds->GetBodyInstance()->bLockXRotation = true;
    m_Bounds->GetBodyInstance()->bLockYRotation = true;
    m_Bounds->GetBodyInstance()->bLockZRotation = true;
    m_Bounds->GetBodyInstance()->bLockXTranslation = true;
    m_Bounds->GetBodyInstance()->bLockYTranslation = true;
    RootComponent = m_Bounds;
    m_Mesh = CreateDefaultSubobject<UStaticMeshComponent>("Mesh");
    m_Mesh->SetupAttachment(RootComponent);
    m_NighbourChecker = CreateDefaultSubobject<USphereComponent>("NeighbourChecker");
    m_NighbourChecker->SetupAttachment(RootComponent);
    m_NighbourChecker->SetSimulatePhysics(false);
    m_NighbourChecker->SetCollisionProfileName("QuerryOnly");
    m_NighbourChecker->SetCollisionEnabled(ECollisionEnabled::QueryOnly);
    m_Colors.SetNum(3);

    m_ColorIndex = EColorEnum::VE_Grey;
    m_Active = true;

    //SetReplicateMovement(true);
    SetReplicates(true);
}

// Called when the game starts or when spawned
void AColorBlock::BeginPlay()
{
	Super::BeginPlay();

    if (Role == ROLE_Authority)
    {
        m_Bounds->SetSimulatePhysics(true);
        m_Bounds->SetCollisionEnabled(ECollisionEnabled::PhysicsOnly);
        m_ColorIndex = (EColorEnum)FMath::RandRange((int)EColorEnum::VE_Red, (int)EColorEnum::VE_Blue);
    }
    else
    {
        //Disable anything related to physics
        m_Bounds->SetSimulatePhysics(false);
        m_Bounds->SetCollisionProfileName("NoCollision");
        m_Bounds->SetCollisionEnabled(ECollisionEnabled::NoCollision);
        m_Bounds->SetSimulatePhysics(false);
        m_Mesh->SetCollisionProfileName("NoCollision");
        m_Bounds->SetCollisionEnabled(ECollisionEnabled::NoCollision);
    }
    m_Mesh->SetMaterial(0, m_Colors[(int)m_ColorIndex]);
}

// Called every frame
void AColorBlock::Tick(float DeltaTime)
{
	Super::Tick(DeltaTime);
}

void AColorBlock::CascadeHit(ALuisMFinalCharacter* a_player)
{
    m_Active = false;
    TArray<AActor*> neighbours;
    TSubclassOf<AActor> templateFilter;
    m_NighbourChecker->GetOverlappingActors(neighbours);

    for (int i = 0; i<neighbours.Num(); i++)
    {
        AColorBlock* block = Cast<AColorBlock>(neighbours[i]);
        if (block != nullptr && block->m_Active == true && block->m_ColorIndex == m_ColorIndex)
        {
            block->CascadeHit(a_player);
        }
    }

    a_player->IncreaseScore();
    Destroy();
}

void AColorBlock::GetLifetimeReplicatedProps(TArray< FLifetimeProperty > & OutLifetimeProps) const
{
    Super::GetLifetimeReplicatedProps(OutLifetimeProps);

    DOREPLIFETIME(AColorBlock, m_ColorIndex);
}