V2EX = way to explore
V2EX 是一个关于分享和探索的地方
现在注册
已注册用户请  登录
NGINX
NGINX Trac
3rd Party Modules
Security Advisories
CHANGES
OpenResty
ngx_lua
Tengine
在线学习资源
NGINX 开发从入门到精通
NGINX Modules
ngx_echo
kmdd33
V2EX  ›  NGINX

深夜处理 bug:重启 openresty(nginx)后, 403 forbidden,连服务器 ip 都无法访问了

  •  
  •   kmdd33 · 2018-06-29 03:02:36 +08:00 · 6860 次点击
    这是一个创建于 2100 天前的主题,其中的信息可能已经有所发展或是发生改变。

    手贱重启了 openresty,突然站点就 403 forbidden。搞了一天了,真心累,请教同学们。

    配置环境:centos7+openresty(nginx)+php+mysql systemctl status openresty ,systemctl status php-fpm,systemctl status mysql,都是 active status,绿色。

    lsof -i :80

    COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME

    nginx 32255 root 6u IPv4 170338 0t0 TCP localhost:http (LISTEN)

    nginx 32332 root 6u IPv4 170338 0t0 TCP localhost:http (LISTEN)

    netstat -an |grep 80

    tcp 0 0 127.0.0.1:80 0.0.0.0:* LISTEN

    unix 2 [ ACC ] STREAM LISTENING 28071 /var/lib/mysql/mysql.sock

    unix 3 [ ] STREAM CONNECTED 11800

    unix 3 [ ] STREAM CONNECTED 11801 /run/systemd/journal/stdout

    ps aux | grep php

    root 19363 0.0 3.1 470292 31572 ? Ss 00:20 0:04 php-fpm: master process (/etc/ph-fpm.conf)

    nginx 19364 0.0 0.7 470292 7228 ? S 00:20 0:00 php-fpm: pool www

    nginx 19365 0.0 0.7 470292 7228 ? S 00:20 0:00 php-fpm: pool www

    nginx 19366 0.0 0.7 470292 7228 ? S 00:20 0:00 php-fpm: pool www

    nginx 19367 0.0 0.7 470292 7228 ? S 00:20 0:00 php-fpm: pool www

    nginx 19368 0.0 0.7 470292 7232 ? S 00:20 0:00 php-fpm: pool www

    nginx 19369 0.0 0.7 470292 7232 ? S 00:20 0:00 php-fpm: pool www

    nginx 19370 0.0 0.7 470292 7232 ? S 00:20 0:00 php-fpm: pool www

    nginx 19371 0.0 0.7 470292 7232 ? S 00:20 0:00 php-fpm: pool www

    root 32204 0.0 0.0 112704 968 pts/0 R+ 17:57 0:00 grep --color=auto php

    netstat -plant

    Active Internet connections (servers and established)

    Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name

    tcp 0 0 127.0.0.1:80 0.0.0.0:* LISTEN 32255/nginx: master

    tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 730/sshd

    tcp 0 0 45.79.66.57:22 113.118.253.226:4315 ESTABLISHED 32434/sshd: root@pt

    tcp 0 36 45.79.66.57:22 113.118.253.226:4860 ESTABLISHED 32584/sshd: root@pt

    tcp 0 0 45.79.66.57:22 113.118.253.226:3505 ESTABLISHED 32115/sshd: root@pt

    tcp6 0 0 :::9000 :::* LISTEN 19363/php-fpm: mast

    tcp6 0 0 :::3306 :::* LISTEN 3299/mysqld

    tcp6 0 0 :::22 :::* LISTEN 730/sshd

    /usr/local/openresty/nginx/conf/nginx.conf

    user root;

    worker_processes 1;

    #error_log logs/error.log;

    #error_log logs/error.log notice;

    #error_log logs/error.log info;

    #pid logs/nginx.pid;

    events { worker_connections 1024; }

    http { include mime.types; default_type application/octet-stream;

    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
    #                  '$status $body_bytes_sent "$http_referer" '
    #                  '"$http_user_agent" "$http_x_forwarded_for"';
    
    #access_log  logs/access.log  main;
    
    sendfile        on;
    #tcp_nopush     on;
    
    #keepalive_timeout  0;
    keepalive_timeout  65;
    
    client_max_body_size 8m;    #允许客户端请求的最大单文件字节数
    client_body_buffer_size 2m;  #缓冲区代理缓冲用户端请求的最大字节
    
    fastcgi_buffers 8 16k;
    fastcgi_buffer_size 32k;
    fastcgi_connect_timeout 300;
    fastcgi_send_timeout 300;
    fastcgi_read_timeout 300;
    
    gzip  on;
    #WAF
    lua_shared_dict limit 50m;
    lua_shared_dict guard_dict 100m;
    lua_shared_dict dict_captcha 70m;
    lua_package_path "/usr/local/openresty/nginx/conf/waf/?.lua";
    init_by_lua_file "/usr/local/openresty/nginx/conf/waf/init.lua";
    access_by_lua_file "/usr/local/openresty/nginx/conf/waf/access.lua";
    
    server {
        listen       127.0.0.1;
        server_name  kiwikiwifly.com;
    
        #charset koi8-r;
    
        #access_log  logs/host.access.log  main;
    
        location / {
            root   html;
            index  index.php index.html index.htm;
        }
    
        #error_page  404              /404.html;
    
        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }
    
        # proxy the PHP scripts to Apache listening on 127.0.0.1:80
        #
        #location ~ \.php$ {
        #    proxy_pass   http://127.0.0.1;
        #}
    
        # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
        #
        
        location ~ \.php$ {
            root           html;
            fastcgi_pass   127.0.0.1:9000;
            fastcgi_index  index.php;
            fastcgi_param  SCRIPT_FILENAME  $document_root$fastcgi_script_name;
            include        fastcgi_params;
        }
    
        # deny access to .htaccess files, if Apache's document root
        # concurs with nginx's one
        #
        #location ~ /\.ht {
        #    deny  all;
        #}
    }
    
    
    # another virtual host using mix of IP-, name-, and port-based configuration
    #
    #server {
    #    listen       8000;
    #    listen       somename:8080;
    #    server_name  somename  alias  another.alias;
    
    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}
    
    
    # HTTPS server
    #
    #server {
    #    listen       443 ssl;
    #    server_name  localhost;
    
    #    ssl_certificate      cert.pem;
    #    ssl_certificate_key  cert.key;
    
    #    ssl_session_cache    shared:SSL:1m;
    #    ssl_session_timeout  5m;
    
    #    ssl_ciphers  HIGH:!aNULL:!MD5;
    #    ssl_prefer_server_ciphers  on;
    
    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}
    

    }

    and /usr/local/openresty/nginx/logs/error.log

    2018/06/26 21:27:10 [alert] 740#740: *7927 open socket #10 left in connection 12

    2018/06/26 21:27:10 [alert] 740#740: *7926 open socket #3 left in connection 13

    2018/06/26 21:27:10 [alert] 740#740: aborting

    只有到 26 号 error.log

    10 条回复    2018-06-29 18:02:42 +08:00
    Livid
        1
    Livid  
    MOD
       2018-06-29 03:06:20 +08:00 via iPhone
    1. 给 server 那段单独写一个 error_log
    2. 试试把 WAF 相关的配置先注释掉
    kmdd33
        2
    kmdd33  
    OP
       2018-06-29 05:40:35 +08:00
    @Livid 2.已经注释了,1 给 server 那段单独写一个 error_log ?请问这个如何操作?
    yaxin
        3
    yaxin  
       2018-06-29 07:54:23 +08:00 via iPhone
    看一下 nginx 有没有 web 根路径的读权限,如果有,在根路径下创建一个 xxxx.php 文件,里面随便输出点字符串,然后访问,看能否访问,能,说明你站点 php 逻辑问题,不能,nginx 问题,通过日志把问题,还是找不到,直接上 strace
    est
        4
    est  
       2018-06-29 09:06:09 +08:00
    livid 的办法是正解。

    开 error_log,debug 级别。就能看到为啥出错了。
    sajesemuy
        5
    sajesemuy  
       2018-06-29 09:37:56 +08:00
    你这边怎么又是 403,另一个帖子 502。listen 127.0.0.1,你是直接 在本地 curl 127.0.0.1 出现 403/502 ?
    如果是访问你的站点的公网地址,前面那一层怎么访问 127.0.0.1 ?
    wqsfree
        6
    wqsfree  
       2018-06-29 09:43:41 +08:00
    user nobody nobody;
    worker_processes 2;
    error_log /var/log/nginx/error.log debug ; 这是错误日志的路径,会了么?
    pid /var/run/nginx/nginx.pid;
    kmdd33
        7
    kmdd33  
    OP
       2018-06-29 16:14:12 +08:00
    @Livid @sajesemuy @est Sorry,Sorry,是 502 bad gateway 的提示。
    kmdd33
        8
    kmdd33  
    OP
       2018-06-29 17:21:05 +08:00
    @wqsfree

    你好,按照您的建议,把你提供的这个写在了 http 段外面,重新 reload openresty 成功,error_log /var/log/nginx/error.log 内容:

    2018/06/29 09:06:08 [notice] 10008#10008: using the "epoll" event method

    2018/06/29 09:06:08 [notice] 10008#10008: openresty/1.13.6.2

    2018/06/29 09:06:08 [notice] 10008#10008: built by gcc 4.8.5 20150623 (Red Hat 4.8.5-16) (GCC)

    2018/06/29 09:06:08 [notice] 10008#10008: OS: Linux 3.10.0-862.3.2.el7.x86_64

    2018/06/29 09:06:08 [notice] 10008#10008: getrlimit(RLIMIT_NOFILE): 1024:4096

    2018/06/29 09:06:08 [notice] 10009#10009: start worker processes

    2018/06/29 09:06:08 [notice] 10009#10009: start worker process 10011

    2018/06/29 09:14:52 [notice] 10009#10009: signal 1 (SIGHUP) received from 10206, reconfiguring

    2018/06/29 09:14:52 [notice] 10009#10009: reconfiguring

    2018/06/29 09:14:52 [notice] 10009#10009: using the "epoll" event method

    2018/06/29 09:14:52 [notice] 10009#10009: start worker processes

    2018/06/29 09:14:52 [notice] 10009#10009: start worker process 10207

    2018/06/29 09:14:52 [notice] 10011#10011: gracefully shutting down

    2018/06/29 09:14:52 [notice] 10011#10011: exiting

    2018/06/29 09:14:52 [notice] 10011#10011: exit

    2018/06/29 09:14:52 [notice] 10009#10009: signal 17 (SIGCHLD) received from 10011

    2018/06/29 09:14:52 [notice] 10009#10009: worker process 10011 exited with code 0

    2018/06/29 09:14:52 [notice] 10009#10009: signal 29 (SIGIO) received

    2018/06/29 09:14:58 [notice] 10009#10009: signal 1 (SIGHUP) received from 10209, reconfiguring

    2018/06/29 09:14:58 [notice] 10009#10009: reconfiguring

    2018/06/29 09:14:58 [notice] 10009#10009: using the "epoll" event method

    2018/06/29 09:14:58 [notice] 10009#10009: start worker processes

    2018/06/29 09:14:58 [notice] 10009#10009: start worker process 10210

    2018/06/29 09:14:58 [notice] 10207#10207: gracefully shutting down

    2018/06/29 09:14:58 [notice] 10207#10207: exiting

    2018/06/29 09:14:58 [notice] 10207#10207: exit

    2018/06/29 09:14:58 [notice] 10009#10009: signal 17 (SIGCHLD) received from 10207

    2018/06/29 09:14:58 [notice] 10009#10009: worker process 10207 exited with code 0

    2018/06/29 09:14:58 [notice] 10009#10009: signal 29 (SIGIO) received
    kmdd33
        9
    kmdd33  
    OP
       2018-06-29 17:23:05 +08:00
    @wqsfree
    /var/run/nginx/nginx.pid 仅仅输出了

    10009

    这个提示。请问我应该修改什么地方呢?服务器 ip 还是无法访问
    kmdd33
        10
    kmdd33  
    OP
       2018-06-29 18:02:42 +08:00
    @Livid @est @sajesemuy @wqsfree @yaxin 最新的 /usr/local/openresty/nginx/log/error.log


    8/06/28 18:04:18 [notice] 32331#32331: signal process started
    2018/06/28 22:19:32 [emerg] 2711#2711: bind() to 127.0.0.1:80 failed (98: Address already in use)
    2018/06/28 22:19:32 [emerg] 2711#2711: bind() to 127.0.0.1:80 failed (98: Address already in use)
    2018/06/28 22:19:32 [emerg] 2711#2711: bind() to 127.0.0.1:80 failed (98: Address already in use)
    2018/06/28 22:19:32 [emerg] 2711#2711: bind() to 127.0.0.1:80 failed (98: Address already in use)
    2018/06/28 22:19:32 [emerg] 2711#2711: bind() to 127.0.0.1:80 failed (98: Address already in use)
    2018/06/28 22:19:32 [emerg] 2711#2711: still could not bind()
    2018/06/29 08:38:40 [emerg] 9553#9553: open() "/var/run/nginx/nginx.pid" failed (2: No such file or directory)
    2018/06/29 08:39:35 [emerg] 9589#9589: open() "/var/run/nginx/nginx.pid" failed (2: No such file or directory)
    2018/06/29 08:51:43 [emerg] 9757#9757: open() "/var/log/nginx/error.log" failed (2: No such file or directory)
    2018/06/29 08:55:53 [notice] 9801#9801: signal process started
    2018/06/29 08:55:53 [error] 9801#9801: invalid PID number "" in "/var/run/nginx/nginx.pid"
    2018/06/29 09:09:34 [notice] 10112#10112: signal process started
    2018/06/29 09:09:34 [error] 10112#10112: invalid PID number "" in "/usr/local/openresty/nginx/logs/nginx.pid"
    2018/06/29 09:14:52 [notice] 10206#10206: signal process started
    2018/06/29 09:14:58 [notice] 10209#10209: signal process started
    2018/06/29 09:30:18 [notice] 10444#10444: signal process started
    2018/06/29 09:32:03 [notice] 10470#10470: signal process started
    2018/06/29 09:44:49 [notice] 10614#10614: signal process started
    2018/06/29 09:44:49 [alert] 10614#10614: kill(10009, 1) failed (3: No such process)
    2018/06/29 09:45:05 [notice] 10616#10616: signal process started
    2018/06/29 09:45:05 [alert] 10616#10616: kill(10009, 1) failed (3: No such process)
    2018/06/29 09:50:38 [notice] 10681#10681: signal process started
    2018/06/29 09:50:38 [alert] 10681#10681: kill(10009, 1) failed (3: No such process)
    关于   ·   帮助文档   ·   博客   ·   API   ·   FAQ   ·   我们的愿景   ·   实用小工具   ·   5872 人在线   最高记录 6543   ·     Select Language
    创意工作者们的社区
    World is powered by solitude
    VERSION: 3.9.8.5 · 27ms · UTC 02:25 · PVG 10:25 · LAX 19:25 · JFK 22:25
    Developed with CodeLauncher
    ♥ Do have faith in what you're doing.