Add support for MikroTik RB5009UG

For Developers
284

[ 8.388007] RAMDISK: image too big! (5484KiB/4096KiB)

WTF: The freaking default is still just 4MB. Can you remind me what year we have? Argh...

Voila, now we're talking.

BusyBox v1.34.1 (2022-01-09 11:08:07 UTC) built-in shell (ash)

  _______                     ________        __
 |       |.-----.-----.-----.|  |  |  |.----.|  |_
 |   -   ||  _  |  -__|     ||  |  |  ||   _||   _|
 |_______||   __|_____|__|__||________||__|  |____|
          |__| W I R E L E S S   F R E E D O M
 -----------------------------------------------------
 OpenWrt SNAPSHOT, r18532-474df71bf0
 -----------------------------------------------------
=== WARNING! =====================================
There is no root password defined on this device!
Use the "passwd" command to set up a new password
in order to prevent unauthorized SSH logins.
--------------------------------------------------
root@(none):/# uname -a
Linux (none) 5.16.0-next-20220114-dirty #36 SMP PREEMPT Fri Jan 14 23:41:50 CET 2022 aarch64 GNU/Linux
root@(none):/#

Just gona have to integrate your switch reset now. Let's see...

285

I doubt it will work, cause for example the GPIO 9/10 is set to GPIO instead of MDIO by default as CPU_MGD is configured as 1 by the pin.

286

Well, isn't that called pinctrl? I should just about get that going...

287

Well, kind of, it's the switch property.
This will sort it out:
mdio f212a200.mdio-mii 0x1c 0x1a 0x8200

Note that you gotta have the mdio-netlink/tools packaged until the driver is adapted to this as the default SMI pinctrl in it doesn't work for Amethyst

288

Yeah, I, of course, have that sucker already in there...

289

Then great, for me it still doesn't work with the QCA8081 driver at all.
With the generic one it does, weird.

Anyway, I would really like to get networking working in U-boot but gotta go watch some Youtube and sleep, traveling again tomorrow so I doubt that I will be working on RB5009

290

No problem, I will concentrate on Linux first. Let's see how it goes...

And have safe travels!

291

Thanks, let me just share my current 5.15 OpenWrt DTS.

// SPDX-License-Identifier: GPL-2.0-or-later OR MIT

/dts-v1/;

#include "armada-7040.dtsi"
#include <dt-bindings/gpio/gpio.h>
#include <dt-bindings/input/input.h>

/ {
	model = "MikroTik RB5009";
	compatible = "mikrotik,rb5009", "marvell,armada7040",
		     "marvell,armada-ap806-quad", "marvell,armada-ap806";

	chosen {
		stdout-path = "serial0:115200n8";
	};

	memory@0 {
		device_type = "memory";
		reg = <0x0 0x0 0x0 0x40000000>;
	};

	aliases {
		led-boot = &led_user;
		led-failsafe = &led_user;
		led-running = &led_user;
		led-upgrade = &led_user;
	};

	sfp_i2c: sfp-i2c {
		compatible = "i2c-gpio";
		sda-gpios = <&cp0_gpio1 0 (GPIO_ACTIVE_HIGH|GPIO_OPEN_DRAIN)>;
		scl-gpios = <&cp0_gpio1 1 (GPIO_ACTIVE_HIGH|GPIO_OPEN_DRAIN)>;
	};

	keys {
		compatible = "gpio-keys";

		reset {
			label = "reset";
			gpios = <&cp0_gpio1 28 GPIO_ACTIVE_LOW>;
			linux,code = <KEY_RESTART>;
		};
	};

	leds {
		compatible = "gpio-leds";

		led_user: user {
			label = "green:user";
			gpios = <&cp0_gpio2 26 GPIO_ACTIVE_LOW>;
		};

		sfp {
			label = "green:sfp";
			gpios = <&cp0_gpio2 25 GPIO_ACTIVE_LOW>;
		};

		hdr1 {
			label = "blue:hdr1";
			gpios = <&cp0_gpio1 4 GPIO_ACTIVE_LOW>;
		};

		hdr2 {
			label = "blue:hdr2";
			gpios = <&cp0_gpio2 19 GPIO_ACTIVE_LOW>;
		};
	};

	sfp: sfp {
		compatible = "sff,sfp";
		i2c-bus = <&sfp_i2c>;
	};
};

&uart0 {
	status = "okay";

	pinctrl-0 = <&uart0_pins>;
	pinctrl-names = "default";
};


&spi0 {
	status = "okay";

	spi-flash@0 {
		#address-cells = <1>;
		#size-cells = <1>;
		compatible = "jedec,spi-nor";
		reg = <0>;
		spi-max-frequency = <20000000>;

		partitions {
			compatible = "fixed-partitions";
			#address-cells = <1>;
			#size-cells = <1>;

			partition@0 {
				label = "firmware";
				reg = <0x0 0x400000>;
			};

			partition@400000 {
				label = "u-boot-env";
				reg = <0x400000 0x20000>;
			};
		};
	};
};

&cp0_nand_controller {
	status = "okay";

	nand@0 {
		reg = <0>;
		nand-rb = <0>;
		nand-ecc-mode = "hw";
		nand-ecc-strength = <4>;
		nand-ecc-step-size = <512>;
		nand-on-flash-bbt;

		partitions {
			compatible = "fixed-partitions";
			#address-cells = <1>;
			#size-cells = <1>;

			partition@0 {
				label = "ubi";
				reg = <0x0 0x40000000>;
			};
		};
	};
};

&cp0_gpio2 {
	enable-usb-power {
		gpio-hog;
		gpios = <23 GPIO_ACTIVE_HIGH>;
		output-high;
		line-name = "enable USB power";
	};

	enable-leds-power {
		gpio-hog;
		gpios = <27 GPIO_ACTIVE_HIGH>;
		output-high;
		line-name = "enable LED-s power";
	};
};

&cp0_usb3_1 {
	status = "okay";
};

&cp0_i2c0 {
	status = "okay";
	clock-frequency = <100000>;
};

&cp0_mdio {
	status = "okay";
};

&cp0_ethernet {
	status = "okay";
};

&cp0_eth0 {
	/* This port is connected to 88E6393X switch */
	status = "okay";
	phy-mode = "10gbase-r";
	managed = "in-band-status";
};

&cp0_mdio {
	status = "okay";

	switch@0 {
		/* Actual device is MV88E6393X */
		compatible = "marvell,mv88e6190";
		#address-cells = <1>;
		#size-cells = <0>;
		reg = <0>;

		ports {
			#address-cells = <1>;
			#size-cells = <0>;

			port@0 {
				reg = <0>;
				label = "cpu";
				ethernet = <&cp0_eth0>;
			};

			port@1 {
				reg = <1>;
				label = "p8";
				phy-handle = <&switch0phy1>;
			};

			port@2 {
				reg = <2>;
				label = "p7";
				phy-handle = <&switch0phy2>;
			};

			port@3 {
				reg = <3>;
				label = "p6";
				phy-handle = <&switch0phy3>;
			};

			port@4 {
				reg = <4>;
				label = "p5";
				phy-handle = <&switch0phy4>;
			};

			port@5 {
				reg = <5>;
				label = "p4";
				phy-handle = <&switch0phy5>;
			};

			port@6 {
				reg = <6>;
				label = "p3";
				phy-handle = <&switch0phy6>;
			};

			port@7 {
				reg = <7>;
				label = "p2";
				phy-handle = <&switch0phy7>;
			};

			port@9 {
				reg = <9>;
				label = "p1";
				phy-mode = "sgmii";
				managed = "in-band-status";
				phy-handle = <&qca8081>;
			};

			port@a {
				reg = <10>;
				label = "sfp";
				phy-mode = "10gbase-r";
				managed = "in-band-status";
				sfp = <&sfp>;
			};
		};

		mdio {
			#address-cells = <1>;
			#size-cells = <0>;

			switch0phy1: switch0phy1@1 {
				reg = <0x1>;
			};

			switch0phy2: switch0phy2@2 {
				reg = <0x2>;
			};

			switch0phy3: switch0phy3@3 {
				reg = <0x3>;
			};

			switch0phy4: switch0phy4@4 {
				reg = <0x4>;
			};

			switch0phy5: switch0phy5@5 {
				reg = <0x5>;
			};

			switch0phy6: switch0phy6@6 {
				reg = <0x6>;
			};

			switch0phy7: switch0phy7@7 {
				reg = <0x7>;
			};
		};

		mdio1 {
			compatible = "marvell,mv88e6xxx-mdio-external";
			#address-cells = <1>;
			#size-cells = <0>;

			qca8081: qca8081@0 {
				reg = <0>;
			};
		};
	};
};
1 Like
292

@sumo Here is the patch to auto-set the pinctrl:

From 40aa82107e7b1e7f20cbf6e644005c0c606f9f79 Mon Sep 17 00:00:00 2001
From: Robert Marko <robimarko@gmail.com>
Date: Fri, 14 Jan 2022 15:17:37 +0100
Subject: [PATCH] net: dsa: mv88e6xxx: add Amethyst specific SMI GPIO function

The existing mv88e6xxx_g2_scratch_gpio_set_smi() cannot be used on the
88E6393X as it requires certain P0_MODE, it also checks the CPU mode
as it impacts the bit setting value.

This is all irrelevant for Amethyst (MV88E6191X/6193X/6393X) as only
the default value of the SMI_PHY Config bit is set to CPU_MGD bootstrap
pin value but it can be changed without restrictions so that GPIO pins
9 and 10 are used as SMI pins.

So, introduce Amethyst specific function and call that if the Amethyst
family wants to setup the external PHY.

Signed-off-by: Robert Marko <robimarko@gmail.com>
---
 drivers/net/dsa/mv88e6xxx/chip.c            |  5 +++-
 drivers/net/dsa/mv88e6xxx/global2.h         |  2 ++
 drivers/net/dsa/mv88e6xxx/global2_scratch.c | 31 +++++++++++++++++++++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c
index 43d126628610..b3803101d612 100644
--- a/drivers/net/dsa/mv88e6xxx/chip.c
+++ b/drivers/net/dsa/mv88e6xxx/chip.c
@@ -3409,7 +3409,10 @@ static int mv88e6xxx_mdio_register(struct mv88e6xxx_chip *chip,
 
 	if (external) {
 		mv88e6xxx_reg_lock(chip);
-		err = mv88e6xxx_g2_scratch_gpio_set_smi(chip, true);
+		if (chip->info->family == MV88E6XXX_FAMILY_6393)
+			err = mv88e6393x_g2_scratch_gpio_set_smi(chip, true);
+		else
+			err = mv88e6xxx_g2_scratch_gpio_set_smi(chip, true);
 		mv88e6xxx_reg_unlock(chip);
 
 		if (err)
diff --git a/drivers/net/dsa/mv88e6xxx/global2.h b/drivers/net/dsa/mv88e6xxx/global2.h
index f3e27573a386..e0649f21f28e 100644
--- a/drivers/net/dsa/mv88e6xxx/global2.h
+++ b/drivers/net/dsa/mv88e6xxx/global2.h
@@ -370,6 +370,8 @@ extern const struct mv88e6xxx_gpio_ops mv88e6352_gpio_ops;
 
 int mv88e6xxx_g2_scratch_gpio_set_smi(struct mv88e6xxx_chip *chip,
 				      bool external);
+int mv88e6393x_g2_scratch_gpio_set_smi(struct mv88e6xxx_chip *chip,
+				       bool external);
 int mv88e6xxx_g2_atu_stats_set(struct mv88e6xxx_chip *chip, u16 kind, u16 bin);
 int mv88e6xxx_g2_atu_stats_get(struct mv88e6xxx_chip *chip, u16 *stats);
 
diff --git a/drivers/net/dsa/mv88e6xxx/global2_scratch.c b/drivers/net/dsa/mv88e6xxx/global2_scratch.c
index eda710062933..dc3f4645fa52 100644
--- a/drivers/net/dsa/mv88e6xxx/global2_scratch.c
+++ b/drivers/net/dsa/mv88e6xxx/global2_scratch.c
@@ -289,3 +289,34 @@ int mv88e6xxx_g2_scratch_gpio_set_smi(struct mv88e6xxx_chip *chip,
 
 	return mv88e6xxx_g2_scratch_write(chip, misc_cfg, val);
 }
+
+/**
+ * mv88e6393x_g2_scratch_gpio_set_smi - set gpio muxing for external smi
+ * @chip: chip private data
+ * @external: set mux for external smi, or free for gpio usage
+ *
+ * MV88E6191X/6193X/6393X GPIO pins 9 and 10 can be configured as an
+ * external SMI interface or as regular GPIO-s.
+ *
+ * They however have a different register layout then the existing
+ * function.
+ */
+
+int mv88e6393x_g2_scratch_gpio_set_smi(struct mv88e6xxx_chip *chip,
+				       bool external)
+{
+	int misc_cfg = MV88E6352_G2_SCRATCH_MISC_CFG;
+	int err;
+	u8 val;
+
+	err = mv88e6xxx_g2_scratch_read(chip, misc_cfg, &val);
+	if (err)
+		return err;
+
+	if (external)
+		val &= ~MV88E6352_G2_SCRATCH_MISC_CFG_NORMALSMI;
+	else
+		val |= MV88E6352_G2_SCRATCH_MISC_CFG_NORMALSMI;
+
+	return mv88e6xxx_g2_scratch_write(chip, misc_cfg, val);
+}
-- 
2.34.1

Gotta cleanup my OpenWrt tree a bit, its a mess now.
Anyway, off to sleep now

293

I tried to specify the reset you found:

switch@0 {
        ...
		reset-gpios = <&cp0_gpio2 2 GPIO_ACTIVE_HIGH>;
        ...
}

The switch itself after the reset works fine, but the LEDs pinout map is lost.
It looks like it is setuped by the RouterBoot at startup. We need to find out which register of the switch chip is responsible for this. What does your datasheet say about LEDs on this switch ?

295

It's not just the LED pinout, it lacks SMI pinctrl as CPU_MGD is set properly, so it means that the PHY-s are by default off as well and port 9 (The 2.5G one) has wrong C_MODE etc.

So, this is just the tip of what needs to be configured and that I found yesterday.

PHY and Serdes registers are separate datasheets with only 152 pages on top of the 714 of the switch datasheet.

It also looks like the SERDES is by default turned off on boot and to enable it you must use C45 via the internal MDIO.

This means that some kind of a switch driver is must-have for U-boot as well

1 Like
296

Now the U-boot is driving me crazy.
I want to get the MDIO bus to automate the switch setup, but for whatever reason it will panic if mvmdio
is the returned uclass device.
If I request any other device, then it doesn't panic.

Like WTF?

"Synchronous Abort" handler, esr 0x96000004
elr: 0000000000066ef0 lr : 000000000001b388 (reloc)
elr: 000000003ffa1ef0 lr : 000000003ff56388
x0 : d503201f1400001a x1 : 000000003fb44360
x2 : 00000000ffffffbd x3 : 0000000000000000
x4 : d503201f1400001a x5 : 0000000000000020
x6 : 0000000000000008 x7 : 000000003fb44360
x8 : 000000003ffc1200 x9 : 0000000000000008
x10: 0000000000001290 x11: 000000003fb1ab1c
x12: 0000000000001240 x13: 000000003fb1abdc
x14: 000000003ffc1200 x15: 0000000000000020
x16: 000000003ffa5bec x17: 0000000000000000
x18: 000000003fb1add0 x19: d503201f1400000a
x20: 000000003ffe4d18 x21: 000000003fb44360
x22: 0000000000000000 x23: 0000000000000000
x24: 0000000000000000 x25: 0000000000000000
x26: 0000000000000000 x27: 0000000000000000
x28: 0000000000000000 x29: 000000003fb1ac90

Code: 3900007f d65f03c0 aa0003e4 d2800003 (38636885) 
Resetting CPU ...

And the code couldn't be any simpler:

int rb5009_mdio_get(void)
{
	struct udevice *dev;
	int ret;

	ret = uclass_get_device_by_driver(UCLASS_MDIO, DM_DRIVER_GET(mvmdio), &dev);
	if (ret) {
		pr_err("failed to find MDIO: %d\n", ret);
		return ret;
	}

	return 0;
}
297

Yep, I feel with you. I fought with that stuff on the Linux side until four in the morning. Got the SFP+ going though. It's really a best if a router!

2 Likes
298

I found what was the reason for the non-working network in U-Boot. There, the driver(mvpp2) does not correctly initialize the MPCS and XPCS registers. Now I have achieved a stable start of the network. Later I will write what exactly needs to be changed in the U-Boot sources.

1 Like
299

Awesome, can you share the proper values?

300

Right now I'm working on the U-Boot code. Give me a few hours to tidy it up and share.

301

Awesome, thanks.
Can you just share the raw value so that I can work on the switch with working ethernet controller?
I will keep on working on the switch code.

302 
writel(0x810, port->priv->mpcs_base + port->gop_id * MVPP22_PORT_OFFSET + 0x14c);
writel(0x46, port->priv->xpcs_base + port->gop_id * MVPP22_PORT_OFFSET + 0x0);
writel(0x810, port->priv->mpcs_base + port->gop_id * MVPP22_PORT_OFFSET + 0x14c);
writel(0x46, port->priv->xpcs_base + port->gop_id * MVPP22_PORT_OFFSET + 0x0);
writel(0x46, port->priv->xpcs_base + port->gop_id * MVPP22_PORT_OFFSET + 0x0);
writel(0x3820, port->priv->mpcs_base + port->gop_id * MVPP22_PORT_OFFSET + 0x14);
writel(0x810, port->priv->mpcs_base + port->gop_id * MVPP22_PORT_OFFSET + 0x14c);
writel(0x17, port->priv->mpcs_base + port->gop_id * MVPP22_PORT_OFFSET + 0x14c);

Paste this instead of mvpp2.c->gop_mpcs_mode()

303

Great, thanks.

U-boot is just driving me insane, whenever I want to somehow use the MDIO bus it panics.
But if you use the mdio command then it works just fine.

304

The patch for fix the network in U-Boot is ready:

commit 9e19fcab292666006276fe89cbdeb4cc0fd9adc2
Author: Sergey Sergeev <adron@yapic.net>
Date:   Sun Jan 16 14:28:30 2022 +0200

    This fix is for Marvell PPv2 network controler(working on 10gbase-r speed).
    
    Due to the lack of XPCS register initialization code and partially incorrect
    initialization of the MPCS network controler registers (tested on Mikrotik RB5009
    in conjunction with MV88E6393X) the network does not work correctly.
    The problem manifests itself as an arbitrary delay (0.4-4 sec) for the actual
    data transmission to the network! Accordingly, an almost completely non-working
    network for U-Boot is obtained. The code is backported from a similar Linux driver.
    
    Signed-off-by: Sergey Sergeev <adron@yapic.net>

diff --git a/drivers/net/mvpp2.c b/drivers/net/mvpp2.c
index 4c0a7b0..a4febbf 100644
--- a/drivers/net/mvpp2.c
+++ b/drivers/net/mvpp2.c
@@ -3304,6 +3304,76 @@ static int gop_gpcs_reset(struct mvpp2_port *port, int reset)
 	return 0;
 }
 
+static void gop_pcs_reset_assert(struct mvpp2_port *port)
+{
+	u32 val;
+
+	if (port->priv->hw_version == MVPP21 || port->gop_id != 0)
+		return;
+
+	val = readl(port->priv->mpcs_base + port->gop_id * MVPP22_PORT_OFFSET +
+		PCS_CLOCK_RESET);
+	val &= ~(MAC_CLK_RESET_MASK | RX_SD_CLK_RESET_MASK | TX_SD_CLK_RESET_MASK);
+	val |= CLK_DIV_PHASE_SET_MASK;
+	writel(val, port->priv->mpcs_base + port->gop_id * MVPP22_PORT_OFFSET +
+		PCS_CLOCK_RESET);
+
+	val = readl(port->priv->xpcs_base + port->gop_id * MVPP22_PORT_OFFSET +
+		MVPP22_XPCS_GLOBAL_CFG_0_REG);
+	val &= ~MVPP22_XPCS_PCSRESET;
+	writel(val,	port->priv->xpcs_base + port->gop_id * MVPP22_PORT_OFFSET +
+		MVPP22_XPCS_GLOBAL_CFG_0_REG);
+}
+
+static void gps_pcs_reset_deassert(struct mvpp2_port *port)
+{
+	u32 val;
+
+	if (port->priv->hw_version == MVPP21 || port->gop_id != 0)
+		return;
+
+	/* this code is only for case of PHY_INTERFACE_MODE_10GBASER! */
+	val = readl(port->priv->mpcs_base + port->gop_id * MVPP22_PORT_OFFSET +
+		PCS_CLOCK_RESET);
+	val |= MAC_CLK_RESET_MASK | RX_SD_CLK_RESET_MASK |
+	       TX_SD_CLK_RESET_MASK;
+	val &= ~CLK_DIV_PHASE_SET_MASK;
+	writel(val, port->priv->mpcs_base + port->gop_id * MVPP22_PORT_OFFSET +
+		PCS_CLOCK_RESET);
+}
+
+/* Set the internal mux's to the required PCS in the PI */
+static int gop_xpcs_mode(struct mvpp2_port *port, int num_of_lanes)
+{
+	u32 val;
+	int lane;
+
+	switch (num_of_lanes) {
+	case 1:
+		lane = 0;
+		break;
+	case 2:
+		lane = 1;
+		break;
+	case 4:
+		lane = 2;
+		break;
+	default:
+		return -1;
+	}
+
+	/* configure XG MAC mode */
+	val = readl(port->priv->xpcs_base + port->gop_id * MVPP22_PORT_OFFSET +
+		MVPP22_XPCS_GLOBAL_CFG_0_REG);
+	val &= ~MVPP22_XPCS_PCSMODE_MASK;
+	val &= ~MVPP22_XPCS_LANEACTIVE_MASK;
+	val |= (2 * lane) << MVPP22_XPCS_LANEACTIVE_OFFS;
+	writel(val, port->priv->xpcs_base + port->gop_id * MVPP22_PORT_OFFSET +
+		MVPP22_XPCS_GLOBAL_CFG_0_REG);
+
+	return 0;
+}
+
 static int gop_mpcs_mode(struct mvpp2_port *port)
 {
 	u32 val;
@@ -3445,7 +3515,10 @@ static int gop_port_init(struct mvpp2_port *port)
 		num_of_act_lanes = 2;
 		mac_num = 0;
 		/* configure PCS */
+		gop_pcs_reset_assert(port);
+		gop_xpcs_mode(port, num_of_act_lanes);
 		gop_mpcs_mode(port);
+		gps_pcs_reset_deassert(port);
 		/* configure MAC */
 		gop_xlg_mac_mode_cfg(port, num_of_act_lanes);
1 Like