Need help finding a website connectivity problem

JaredBusch

Here is the sanitized ERL config.

firewall {
    all-ping enable
    broadcast-ping disable
    group {
        network-group Private_LAN {
            description "Private LAN Networks"
            network 10.204.0.0/16
        }
    }
    ipv6-receive-redirects disable
    ipv6-src-route disable
    ip-src-route disable
    log-martians enable
    modify PPPoE_OUT {
        description "TCP clamping"
        rule 1 {
            action modify
            modify {
                tcp-mss 1452
            }
            protocol tcp
            tcp {
                flags SYN
            }
        }
    }
    name LAN_IN {
        default-action accept
        description "Internal network to Internet"
        rule 10 {
            action accept
            description "Allow SMTP to ACIDC01"
            destination {
                address 10.1.1.2/32
                port 25
            }
            log disable
            protocol tcp
            state {
                established enable
                invalid disable
                new enable
                related enable
            }
        }
        rule 20 {
            action drop
            description "Drop All SMTP"
            destination {
                port 25
            }
            log enable
            protocol tcp
            state {
                established enable
                invalid enable
                new enable
                related enable
            }
        }
    }
    name LAN_LOCAL {
        default-action accept
        description "Internal network to router"
    }
    name PPPoE_IN {
        default-action drop
        description "WAN to Internal Networks"
        rule 10 {
            action accept
            state {
                established enable
                related enable
            }
        }
        rule 20 {
            action drop
            log enable
            state {
                invalid enable
            }
        }
    }
    name PPPoE_LOCAL {
        default-action drop
        description "WAN to Router"
        rule 10 {
            action accept
            state {
                established enable
                related enable
            }
        }
        rule 20 {
            action drop
            log enable
            state {
                invalid enable
            }
        }
        rule 50 {
            action accept
            description "ICMP 50/m"
            limit {
                burst 1
                rate 50/minute
            }
            log enable
            protocol icmp
        }
        rule 60 {
            action accept
            description "Accept OpenVPN Connections"
            destination {
                group {
                    address-group ADDRv4_pppoe0
                }
                port 1194
            }
            log disable
            protocol udp
            state {
                established enable
                invalid disable
                new enable
                related enable
            }
        }
    }
    name Public_WiFi_IN {
        default-action accept
        description "Public WiFi to Internet"
        rule 10 {
            action accept
            description "Allow Response from LAN"
            log disable
            protocol all
            state {
                established enable
                invalid disable
                new disable
                related enable
            }
        }
        rule 20 {
            action drop
            description "Block Access to Private Networks"
            destination {
                group {
                    network-group Private_LAN
                }
            }
            log enable
            protocol all
        }
        rule 30 {
            action drop
            description "Block SMTP"
            destination {
                port 25
            }
            log enable
            protocol tcp
        }
    }
    name Public_WiFi_LOCAL {
        default-action drop
        description "Public WiFi to Router"
        rule 10 {
            action accept
            description "Allow DNS"
            destination {
                port 53
            }
            log enable
            protocol udp
        }
        rule 50 {
            action accept
            description "Allow pings"
            limit {
                burst 1
                rate 62/minute
            }
            log enable
            protocol icmp
        }
    }
    receive-redirects disable
    send-redirects enable
    source-validation disable
    syn-cookies enable
}
interfaces {
    ethernet eth0 {
        address 10.204.4.9/29
        description "WiFi Management"
        duplex auto
        firewall {
            in {
                name LAN_IN
            }
            local {
                name LAN_LOCAL
            }
        }
        speed auto
        vif 5 {
            address 10.204.11.1/24
            description "Private WiFi"
            firewall {
                in {
                    name LAN_IN
                }
                local {
                    name LAN_LOCAL
                }
            }
        }
        vif 6 {
            address 10.204.12.1/24
            description "Public WiFi"
            firewall {
                in {
                    name Public_WiFi_IN
                }
                local {
                    name Public_WiFi_LOCAL
                }
            }
        }
    }
    ethernet eth1 {
        address 10.204.10.1/24
        description LAN
        duplex auto
        firewall {
            in {
                name LAN_IN
            }
            local {
                name LAN_LOCAL
            }
        }
        speed auto
    }
    ethernet eth2 {
        description WAN
        duplex auto
        pppoe 0 {
            default-route auto
            firewall {
                in {
                    name PPPoE_IN
                }
                local {
                    name PPPoE_LOCAL
                }
            }
            mtu 1492
            name-server auto
            password XXXXXXXXXXXXXX
            traffic-policy {
                out DSL_up
            }
            user-id XXXXXXXXXXXXXX
        }
        speed auto
    }
    loopback lo {
    }
    openvpn vtun0 {
        description "User OpenVPN Server"
        encryption aes128
        mode server
        openvpn-option --tls-server
        openvpn-option "--proto udp"
        openvpn-option "--port 1194"
        openvpn-option "--tun-mtu 1400"
        openvpn-option --persist-key
        openvpn-option --persist-tun
        openvpn-option --persist-local-ip
        openvpn-option --persist-remote-ip
        openvpn-option "--keepalive 8 30"
        openvpn-option --comp-lzo
        openvpn-option --duplicate-cn
        openvpn-option "--client-cert-not-required --username-as-common-name"
        openvpn-option "--verb 1"
        openvpn-option --client-to-client
        openvpn-option "--user nobody --group nogroup"
        openvpn-option "--push route 10.204.10.0 255.255.255.0"
        openvpn-option "--push route 10.204.11.0 255.255.255.0"
        openvpn-option "--push route 10.204.12.0 255.255.255.0"
        openvpn-option "--push route 10.204.1.0 255.255.255.0"
        openvpn-option "--push route 10.204.5.0 255.255.255.0"
        openvpn-option "--push route 10.204.6.0 255.255.255.0"
        openvpn-option "--push redirect-gateway def1"
        openvpn-option "--push dhcp-option DNS 10.1.1.2"
        openvpn-option "--push dhcp-option DNS 10.204.10.1"
        openvpn-option "--plugin /usr/lib/openvpn/openvpn-plugin-auth-pam.so login"
        server {
            subnet 10.204.13.0/24
            topology subnet
        }
        tls {
            ca-cert-file /config/auth/openvpn/keys/XXXXXXXXXXXXXX.crt
            cert-file /config/auth/openvpn/keys/XXXXXXXXXXXXXX.crt
            dh-file /config/auth/openvpn/keys/XXXXXXXXXXXXXX.pem
            key-file /config/auth/openvpn/keys/XXXXXXXXXXXXXX.key
        }
    }
    openvpn vtun5 {
        description "XXXXXXXXXXXXXX to Jared"
        local-address 10.204.9.5 {
        }
        local-port 1201
        mode site-to-site
        openvpn-option --comp-lzo
        openvpn-option "--tun-mtu 1472"
        remote-address 10.204.9.6
        remote-host jared.bundystl.com
        remote-port 1201
        shared-secret-key-file /config/auth/XXXXXXXXXXXXXX
    }
    openvpn vtun10 {
        description "XXXXXXXXXXXXXX to XXXXXXXXXXXXXX"
        local-address 10.204.9.2 {
        }
        local-port 1195
        mode site-to-site
        openvpn-option --comp-lzo
        openvpn-option "--tun-mtu 1464"
        remote-address 10.204.9.1
        remote-host vpn.XXXXXXXXXXXXXX.com
        remote-port 1195
        shared-secret-key-file /config/auth/XXXXXXXXXXXXXX
    }
}
protocols {
    static {
        interface-route 10.1.1.0/24 {
            next-hop-interface vtun10 {
            }
        }
        interface-route 10.204.1.0/24 {
            next-hop-interface vtun10 {
            }
        }
        interface-route 10.204.5.0/24 {
            next-hop-interface vtun10 {
            }
        }
        interface-route 10.204.6.0/24 {
            next-hop-interface vtun10 {
            }
        }
        interface-route 10.254.103.0/24 {
            next-hop-interface vtun5 {
            }
        }
        interface-route 10.254.203.0/24 {
            next-hop-interface vtun5 {
            }
        }
    }
}
service {
    dhcp-server {
        disabled false
        hostfile-update enable
        shared-network-name XXXXXXXXXXXXXX_LAN {
            authoritative disable
            subnet 10.204.10.0/24 {
                default-router 10.204.10.1
                dns-server 10.1.1.2
                domain-name XXXXXXXXXXXXXX.local
                lease 86400
                start 10.204.10.50 {
                    stop 10.204.10.254
                }
                static-mapping NPID5FA4B {
                    ip-address 10.204.10.11
                    mac-address 2c:59:e5:d5:fa:4b
                }
            }
        }
        shared-network-name Private_WiFi {
            authoritative disable
            subnet 10.204.11.0/24 {
                default-router 10.204.11.1
                dns-server 10.1.1.2
                dns-server 10.204.11.1
                lease 86400
                start 10.204.11.10 {
                    stop 10.204.11.254
                }
            }
        }
        shared-network-name Public_WiFi {
            authoritative disable
            subnet 10.204.12.0/24 {
                default-router 10.204.12.1
                dns-server 10.204.12.1
                lease 3600
                start 10.204.12.10 {
                    stop 10.204.12.254
                }
            }
        }
        shared-network-name WiFi_Management {
            authoritative disable
            subnet 10.204.4.8/29 {
                default-router 10.204.4.9
                dns-server 10.204.10.1
                lease 86400
                start 10.204.4.10 {
                    stop 10.204.4.14
                }
                unifi-controller 207.244.223.13
            }
        }
    }
    dns {
        dynamic {
            interface pppoe0 {
                service afraid {
                    host-name XXXXXXXXXXXXXX
                    login XXXXXXXXXXXXXX
                    password XXXXXXXXXXXXXX
                }
            }
        }
        forwarding {
            cache-size 150
            listen-on eth0
            listen-on eth0.5
            listen-on eth0.6
            listen-on eth1
            listen-on vtun0
            system
        }
    }
    gui {
        https-port 443
        listen-address 10.204.10.1
    }
    nat {
        rule 5010 {
            log disable
            outbound-interface pppoe0
            protocol all
            type masquerade
        }
    }
    snmp {
        community public {
            authorization ro
        }
    }
    ssh {
        listen-address 10.204.10.1
        port 22
        protocol-version v2
    }
    upnp {
        listen-on eth0 {
            outbound-interface pppoe0
        }
        listen-on eth1 {
            outbound-interface pppoe0
        }
    }
}
system {
    domain-name XXXXXXXXXXXXXX.local
    host-name XXXXXXXXXXXXXX
    login {
        user XXXXXXXXXXXXXX {
            authentication {
                encrypted-password XXXXXXXXXXXXXX
                plaintext-password ""
            }
            full-name "ACI Administrator"
            level admin
        }
    }
    name-server 10.1.1.2
    name-server 8.8.8.8
    name-server 8.8.4.4
    ntp {
        server 0.ubnt.pool.ntp.org {
        }
        server 1.ubnt.pool.ntp.org {
        }
        server 2.ubnt.pool.ntp.org {
        }
        server 3.ubnt.pool.ntp.org {
        }
    }
    offload {
        ipv4 {
            forwarding enable
            pppoe enable
            vlan enable
        }
    }
    syslog {
        global {
            facility all {
                level notice
            }
            facility protocols {
                level debug
            }
        }
    }
    time-zone America/Chicago
    traffic-analysis {
        dpi enable
        export enable
    }
}
traffic-policy {
    shaper DSL_up {
        bandwidth 700kbit
        class 10 {
            bandwidth 75%
            burst 15k
            ceiling 100%
            description "DSL up RTP Traffic"
            match IAX2 {
                ip {
                    destination {
                        port 4569
                    }
                }
            }
            match RTP {
                ip {
                    dscp 46
                }
            }
            match RTP-IPv6 {
                ipv6 {
                    dscp 46
                }
            }
            queue-type fair-queue
        }
        class 20 {
            bandwidth 5%
            burst 15k
            ceiling 100%
            description "DSL up SIP Traffic"
            match ICMP {
                ip {
                    protocol ICMP
                }
            }
            match SIP {
                ip {
                    dscp 26
                }
            }
            match SIP-IPv6 {
                ipv6 {
                    dscp 26
                }
            }
            queue-type fair-queue
        }
        default {
            bandwidth 20%
            burst 15k
            ceiling 100%
            queue-type fair-queue
        }
    }
}


/* Warning: Do not remove the following line. */
/* === vyatta-config-version: "config-management@1:conntrack@1:cron@1:dhcp-relay@1:dhcp-server@4:firewall@5:ipsec@4:nat@3:qos@1:quagga@2:system@4:ubnt-pptp@1:ubnt-util@1:vrrp@1:webgui@1:webproxy@1:zone-policy@1" === */
/* Release version: v1.7.0.4783374.150622.1534 */

hubtechagain

for giggles have you tried changing dns on a workstation there to google or open dns and see if anything changes?

PSX_Defector

Drop the MTU from 1492 to 1484 then 1476. See if it works then.

Jason

@PSX_Defector said:

Drop the MTU from 1492 to 1484 then 1476. See if it works then.

That's what I was thinking.

Jason

@hubtechagain said:

for giggles have you tried changing dns on a workstation there to google or open dns and see if anything changes?

DNS is properly resolving and the same at both sites (on SBS) so I don't see any way that could help.

PSX_Defector

@Jason said:

@PSX_Defector said:

Drop the MTU from 1492 to 1484 then 1476. See if it works then.

That's what I was thinking.

The question is, do you know why?

art_of_shred

@PSX_Defector said:

@Jason said:

@PSX_Defector said:

Drop the MTU from 1492 to 1484 then 1476. See if it works then.

That's what I was thinking.

The question is, do you know why?

I don't, but I'd like to. Why the "8" drops?

PSX_Defector

@art_of_shred said:

@PSX_Defector said:

@Jason said:

@PSX_Defector said:

Drop the MTU from 1492 to 1484 then 1476. See if it works then.

That's what I was thinking.

The question is, do you know why?

I don't, but I'd like to. Why the "8" drops?

Time for class folks.

We know the site is up and running, as we can access it via other places. We know it's on Azure because of the trace. The trace tells us another interesting tidbit though. I'm wondering if anyone can see it.

PSX_Defector

@art_of_shred said:

@PSX_Defector said:

@Jason said:

@PSX_Defector said:

Drop the MTU from 1492 to 1484 then 1476. See if it works then.

That's what I was thinking.

The question is, do you know why?

I don't, but I'd like to. Why the "8" drops?

And the drop in 8's is because it's a base8 world. The MTU is the size of the packet in bytes. Odd byte numbers make for a bad time.

Which brings another item. Does anyone know why I went straight for MTU?

JaredBusch

It is PPPoE which impacts the MTU. but there is a firewall rule in place that has been there for a year and supposedly it was working, up until a week or so ago.

JaredBusch

@PSX_Defector said:

@art_of_shred said:

@PSX_Defector said:

@Jason said:

@PSX_Defector said:

Drop the MTU from 1492 to 1484 then 1476. See if it works then.

That's what I was thinking.

The question is, do you know why?

I don't, but I'd like to. Why the "8" drops?

And the drop in 8's is because it's a base8 world. The MTU is the size of the packet in bytes. Odd byte numbers make for a bad time.

Which brings another item. Does anyone know why I went straight for MTU?

I had this thread on the MTU subject 2 weeks ago.

http://mangolassi.it/topic/7118/a-little-confused-on-openvpn-mtu

I made no changes on the pppoe interface though. so I would not know why it would have been a cause (if it is).

I looked at a config backup from July and it is the same for pppoe being 1492

JaredBusch

@PSX_Defector said:

@art_of_shred said:

@PSX_Defector said:

@Jason said:

@PSX_Defector said:

Drop the MTU from 1492 to 1484 then 1476. See if it works then.

That's what I was thinking.

The question is, do you know why?

I don't, but I'd like to. Why the "8" drops?

And the drop in 8's is because it's a base8 world. The MTU is the size of the packet in bytes. Odd byte numbers make for a bad time.

Which brings another item. Does anyone know why I went straight for MTU?

Because it is DSL and DSL is generally PPPoE which takes up another 8 bytes?

JaredBusch

When you use the wizard to setup PPPoE on an ERL is automatically creates this firewall rule and applies it to the out of the PPPoE

modify PPPoE_OUT {
    description "TCP clamping"
    rule 1 {
        action modify
        modify {
            tcp-mss 1452
        }
        protocol tcp
        tcp {
            flags SYN
        }
    }
}

ethernet eth2 {
    description WAN
    duplex auto
    pppoe 0 {
        default-route auto
        firewall {
            in {
                name PPPoE_IN
            }
            local {
                name PPPoE_LOCAL
            }
            out {
                modify PPPoE_OUT
            }
        }
        mtu 1492
        name-server auto
        password 
        user-id 
    }
    speed auto
}

Jason

@PSX_Defector said:

@art_of_shred said:

@PSX_Defector said:

@Jason said:

@PSX_Defector said:

Drop the MTU from 1492 to 1484 then 1476. See if it works then.

That's what I was thinking.

The question is, do you know why?

I don't, but I'd like to. Why the "8" drops?

Time for class folks.

We know the site is up and running, as we can access it via other places. We know it's on Azure because of the trace. The trace tells us another interesting tidbit though. I'm wondering if anyone can see it.

my thought was it was odd that a hop inside the ISP network did not reply. Microsoft not replying is expected.

JaredBusch

@Jason said:

@PSX_Defector said:

@art_of_shred said:

@PSX_Defector said:

@Jason said:

@PSX_Defector said:

Drop the MTU from 1492 to 1484 then 1476. See if it works then.

That's what I was thinking.

The question is, do you know why?

I don't, but I'd like to. Why the "8" drops?

Time for class folks.

We know the site is up and running, as we can access it via other places. We know it's on Azure because of the trace. The trace tells us another interesting tidbit though. I'm wondering if anyone can see it.

my thought was it was odd that a hop inside the ISP network did not reply. Microsoft not replying is expected.

I was concerned about the 10.X.X.X showing in a trace. The site is on 10.204.10.0/24 and I have routes across VPN tunnels to 10.1.1.0/24, a few 10.204.X.0/24 and 10.254.103.0/24 as well.

But the site on the other end of that VPN tunnel also has all that and works fine.

JaredBusch

But bundystl.com is hosted on Azure and if you look at it directly (bundystl.azurewebsites.net), instead of via CloudFlare, it works just fine from on the client site.

it has the same trace results.

JaredBusch

@PSX_Defector setting the MTU down to 1476 makes no difference in the pages loading.

PSX_Defector

@JaredBusch said:

@Jason said:

@PSX_Defector said:

@art_of_shred said:

@PSX_Defector said:

@Jason said:

@PSX_Defector said:

Drop the MTU from 1492 to 1484 then 1476. See if it works then.

That's what I was thinking.

The question is, do you know why?

I don't, but I'd like to. Why the "8" drops?

Time for class folks.

We know the site is up and running, as we can access it via other places. We know it's on Azure because of the trace. The trace tells us another interesting tidbit though. I'm wondering if anyone can see it.

my thought was it was odd that a hop inside the ISP network did not reply. Microsoft not replying is expected.

I was concerned about the 10.X.X.X showing in a trace. The site is on 10.204.10.0/24 and I have routes across VPN tunnels to 10.1.1.0/24, a few 10.204.X.0/24 and 10.254.103.0/24 as well.

But the site on the other end of that VPN tunnel also has all that and works fine.

Ahh, the plot thickens!

I thought it was strange that I couldn't get the same trace, but since you mention that, it makes more sense. The reason I say something about MTU is that I know there is sometimes fun when attempting to access certain sites if they are behind carrier NAT. Remember when SBC flipped over some PoPs to NAT for various stuff between BRAS and edge? I saw wacky routes, slow sites, all kinds of things. Most of it was because idiots were double NAT'ed. But on occasion, I would find a site that would not work without the MTU being 1500.

Now with the VPN tunnel tidbit, we need to make sure we are good. I thought it might have been a problem, but I didn't see it in your screenshots. The scope should be sufficiently small enough to not encompass any of the hops you are hitting. But I would double check that.

This is why I use 172.16.0.0/24 on my network at home. I never see funny shit like this.

JaredBusch

/wtb someone else on CenturyTel to test with.

JaredBusch

@PSX_Defector said:

@JaredBusch said:

@Jason said:

@PSX_Defector said:

@art_of_shred said:

@PSX_Defector said:

@Jason said:

@PSX_Defector said:

Drop the MTU from 1492 to 1484 then 1476. See if it works then.

That's what I was thinking.

The question is, do you know why?

I don't, but I'd like to. Why the "8" drops?

Time for class folks.

We know the site is up and running, as we can access it via other places. We know it's on Azure because of the trace. The trace tells us another interesting tidbit though. I'm wondering if anyone can see it.

my thought was it was odd that a hop inside the ISP network did not reply. Microsoft not replying is expected.

I was concerned about the 10.X.X.X showing in a trace. The site is on 10.204.10.0/24 and I have routes across VPN tunnels to 10.1.1.0/24, a few 10.204.X.0/24 and 10.254.103.0/24 as well.

But the site on the other end of that VPN tunnel also has all that and works fine.

Ahh, the plot thickens!

I thought it was strange that I couldn't get the same trace, but since you mention that, it makes more sense. The reason I say something about MTU is that I know there is sometimes fun when attempting to access certain sites if they are behind carrier NAT. Remember when SBC flipped over some PoPs to NAT for various stuff between BRAS and edge? I saw wacky routes, slow sites, all kinds of things. Most of it was because idiots were double NAT'ed. But on occasion, I would find a site that would not work without the MTU being 1500.

Now with the VPN tunnel tidbit, we need to make sure we are good. I thought it might have been a problem, but I didn't see it in your screenshots. The scope should be sufficiently small enough to not encompass any of the hops you are hitting. But I would double check that.

This is why I use 172.16.0.0/24 on my network at home. I never see funny shit like this.

let me revert the MTU and then shutdown some tunnels.